Merge pull request #3 from vulcanize/vdb-298-pit-diff-parsing

[VDB-298] Parse storage diffs from the Pit contract

cmd/backfillMakerLogs.go

@@ -47,7 +47,7 @@ func backfillMakerLogs() {
 		log.Fatal("Failed to initialize database.")
 	}
 
-	watcher := shared.NewWatcher(db, blockChain)
+	watcher := shared.NewEventWatcher(db, blockChain)
 
 	watcher.AddTransformers(transformers.TransformerInitializers())
 	err = watcher.Execute()

cmd/continuousLogSync.go

@@ -62,7 +62,7 @@ func syncMakerLogs() {
 
 	initializers := getTransformerInitializers(transformerNames)
 
-	watcher := shared.NewWatcher(db, blockChain)
+	watcher := shared.NewEventWatcher(db, blockChain)
 	watcher.AddTransformers(initializers)
 
 	for range ticker.C {

cmd/parseStorageDiffs.go

@@ -0,0 +1,66 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/vulcanize/vulcanizedb/libraries/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/fs"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage"
+	"log"
+)
+
+// parseStorageDiffsCmd represents the parseStorageDiffs command
+var parseStorageDiffsCmd = &cobra.Command{
+	Use:   "parseStorageDiffs",
+	Short: "Continuously ingest storage diffs from a CSV file",
+	Long: `Read storage diffs out of a CSV file that is constantly receiving
+new rows from an Ethereum node. For example:
+
+./vulcanizedb parseStorageDiffs --config=environments/staging.toml
+
+Note that the path to your storage diffs must be configured in your toml
+file under storageDiffsPath.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		parseStorageDiffs()
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(parseStorageDiffsCmd)
+}
+
+func parseStorageDiffs() {
+	blockChain := getBlockChain()
+	db, err := postgres.NewDB(databaseConfig, blockChain.Node())
+	if err != nil {
+		log.Fatal("Failed to initialize database: ", err)
+	}
+
+	tailer := fs.FileTailer{Path: storageDiffsPath}
+
+	// TODO: configure transformers
+	watcher := shared.NewStorageWatcher(tailer, db)
+	watcher.AddTransformers([]storage.TransformerInitializer{transformers.GetPitStorageTransformer().NewTransformer})
+
+	err = watcher.Execute()
+	if err != nil {
+		log.Fatal(err)
+	}
+}

cmd/root.go

@@ -40,6 +40,7 @@ var (
 	ipc                 string
 	levelDbPath         string
 	startingBlockNumber int64
+	storageDiffsPath    string
 	syncAll             bool
 	endingBlockNumber   int64
 )
@@ -64,6 +65,7 @@ func Execute() {
 func database(cmd *cobra.Command, args []string) {
 	ipc = viper.GetString("client.ipcpath")
 	levelDbPath = viper.GetString("client.leveldbpath")
+	storageDiffsPath = viper.GetString("filesystem.storageDiffsPath")
 	databaseConfig = config.Database{
 		Name:     viper.GetString("database.name"),
 		Hostname: viper.GetString("database.hostname"),
@@ -86,6 +88,7 @@ func init() {
 	rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
 	rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
 	rootCmd.PersistentFlags().String("datadog-name", "vulcanize-test", "datadog service name")
+	rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
 
 	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
 	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
@@ -95,6 +98,7 @@ func init() {
 	viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath"))
 	viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
 	viper.BindPFlag("datadog.name", rootCmd.PersistentFlags().Lookup("datadog-name"))
+	viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
 }
 
 func initConfig() {

db/migrations/00071_create_pit_contract_storage_tables.sql

@@ -0,0 +1,52 @@
+-- +goose Up
+CREATE TABLE maker.pit_drip (
+  id            SERIAL PRIMARY KEY,
+  block_number  BIGINT,
+  block_hash    TEXT,
+  drip          TEXT
+);
+
+CREATE TABLE maker.pit_ilk_spot (
+  id            SERIAL PRIMARY KEY,
+  block_number  BIGINT,
+  block_hash    TEXT,
+  ilk           TEXT,
+  spot          NUMERIC NOT NULL
+);
+
+CREATE TABLE maker.pit_ilk_line (
+  id            SERIAL PRIMARY KEY,
+  block_number  BIGINT,
+  block_hash    TEXT,
+  ilk           TEXT,
+  line          NUMERIC NOT NULL
+);
+
+CREATE TABLE maker.pit_line (
+  id            SERIAL PRIMARY KEY,
+  block_number  BIGINT,
+  block_hash    TEXT,
+  line          NUMERIC NOT NULL
+);
+
+CREATE TABLE maker.pit_live (
+  id            SERIAL PRIMARY KEY,
+  block_number  BIGINT,
+  block_hash    TEXT,
+  live          NUMERIC NOT NULL
+);
+
+CREATE TABLE maker.pit_vat (
+  id            SERIAL PRIMARY KEY,
+  block_number  BIGINT,
+  block_hash    TEXT,
+  vat           TEXT
+);
+
+-- +goose Down
+DROP TABLE maker.pit_drip;
+DROP TABLE maker.pit_ilk_spot;
+DROP TABLE maker.pit_ilk_line;
+DROP TABLE maker.pit_line;
+DROP TABLE maker.pit_live;
+DROP TABLE maker.pit_vat;

db/schema.sql

@@ -3,7 +3,7 @@
 --
 
 -- Dumped from database version 10.5
--- Dumped by pg_dump version 11.1
+-- Dumped by pg_dump version 10.5
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -22,6 +22,20 @@ SET row_security = off;
 CREATE SCHEMA maker;
 
 
+--
+-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: -
+--
+
+CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
+
+
+--
+-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: -
+--
+
+COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
+
+
 --
 -- Name: notify_pricefeed(); Type: FUNCTION; Schema: public; Owner: -
 --
@@ -559,6 +573,38 @@ CREATE SEQUENCE maker.frob_id_seq
 ALTER SEQUENCE maker.frob_id_seq OWNED BY maker.frob.id;
 
 
+--
+-- Name: pit_drip; Type: TABLE; Schema: maker; Owner: -
+--
+
+CREATE TABLE maker.pit_drip (
+    id integer NOT NULL,
+    block_number bigint,
+    block_hash text,
+    drip text
+);
+
+
+--
+-- Name: pit_drip_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
+--
+
+CREATE SEQUENCE maker.pit_drip_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: pit_drip_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
+--
+
+ALTER SEQUENCE maker.pit_drip_id_seq OWNED BY maker.pit_drip.id;
+
+
 --
 -- Name: pit_file_debt_ceiling; Type: TABLE; Schema: maker; Owner: -
 --
@@ -630,6 +676,168 @@ CREATE SEQUENCE maker.pit_file_ilk_id_seq
 ALTER SEQUENCE maker.pit_file_ilk_id_seq OWNED BY maker.pit_file_ilk.id;
 
 
+--
+-- Name: pit_ilk_line; Type: TABLE; Schema: maker; Owner: -
+--
+
+CREATE TABLE maker.pit_ilk_line (
+    id integer NOT NULL,
+    block_number bigint,
+    block_hash text,
+    ilk text,
+    line numeric NOT NULL
+);
+
+
+--
+-- Name: pit_ilk_line_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
+--
+
+CREATE SEQUENCE maker.pit_ilk_line_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: pit_ilk_line_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
+--
+
+ALTER SEQUENCE maker.pit_ilk_line_id_seq OWNED BY maker.pit_ilk_line.id;
+
+
+--
+-- Name: pit_ilk_spot; Type: TABLE; Schema: maker; Owner: -
+--
+
+CREATE TABLE maker.pit_ilk_spot (
+    id integer NOT NULL,
+    block_number bigint,
+    block_hash text,
+    ilk text,
+    spot numeric NOT NULL
+);
+
+
+--
+-- Name: pit_ilk_spot_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
+--
+
+CREATE SEQUENCE maker.pit_ilk_spot_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: pit_ilk_spot_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
+--
+
+ALTER SEQUENCE maker.pit_ilk_spot_id_seq OWNED BY maker.pit_ilk_spot.id;
+
+
+--
+-- Name: pit_line; Type: TABLE; Schema: maker; Owner: -
+--
+
+CREATE TABLE maker.pit_line (
+    id integer NOT NULL,
+    block_number bigint,
+    block_hash text,
+    line numeric NOT NULL
+);
+
+
+--
+-- Name: pit_line_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
+--
+
+CREATE SEQUENCE maker.pit_line_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: pit_line_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
+--
+
+ALTER SEQUENCE maker.pit_line_id_seq OWNED BY maker.pit_line.id;
+
+
+--
+-- Name: pit_live; Type: TABLE; Schema: maker; Owner: -
+--
+
+CREATE TABLE maker.pit_live (
+    id integer NOT NULL,
+    block_number bigint,
+    block_hash text,
+    live numeric NOT NULL
+);
+
+
+--
+-- Name: pit_live_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
+--
+
+CREATE SEQUENCE maker.pit_live_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: pit_live_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
+--
+
+ALTER SEQUENCE maker.pit_live_id_seq OWNED BY maker.pit_live.id;
+
+
+--
+-- Name: pit_vat; Type: TABLE; Schema: maker; Owner: -
+--
+
+CREATE TABLE maker.pit_vat (
+    id integer NOT NULL,
+    block_number bigint,
+    block_hash text,
+    vat text
+);
+
+
+--
+-- Name: pit_vat_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
+--
+
+CREATE SEQUENCE maker.pit_vat_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: pit_vat_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
+--
+
+ALTER SEQUENCE maker.pit_vat_id_seq OWNED BY maker.pit_vat.id;
+
+
 --
 -- Name: price_feeds; Type: TABLE; Schema: maker; Owner: -
 --
@@ -1618,6 +1826,13 @@ ALTER TABLE ONLY maker.flop_kick ALTER COLUMN id SET DEFAULT nextval('maker.flop
 ALTER TABLE ONLY maker.frob ALTER COLUMN id SET DEFAULT nextval('maker.frob_id_seq'::regclass);
 
 
+--
+-- Name: pit_drip id; Type: DEFAULT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_drip ALTER COLUMN id SET DEFAULT nextval('maker.pit_drip_id_seq'::regclass);
+
+
 --
 -- Name: pit_file_debt_ceiling id; Type: DEFAULT; Schema: maker; Owner: -
 --
@@ -1632,6 +1847,41 @@ ALTER TABLE ONLY maker.pit_file_debt_ceiling ALTER COLUMN id SET DEFAULT nextval
 ALTER TABLE ONLY maker.pit_file_ilk ALTER COLUMN id SET DEFAULT nextval('maker.pit_file_ilk_id_seq'::regclass);
 
 
+--
+-- Name: pit_ilk_line id; Type: DEFAULT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_ilk_line ALTER COLUMN id SET DEFAULT nextval('maker.pit_ilk_line_id_seq'::regclass);
+
+
+--
+-- Name: pit_ilk_spot id; Type: DEFAULT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_ilk_spot ALTER COLUMN id SET DEFAULT nextval('maker.pit_ilk_spot_id_seq'::regclass);
+
+
+--
+-- Name: pit_line id; Type: DEFAULT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_line ALTER COLUMN id SET DEFAULT nextval('maker.pit_line_id_seq'::regclass);
+
+
+--
+-- Name: pit_live id; Type: DEFAULT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_live ALTER COLUMN id SET DEFAULT nextval('maker.pit_live_id_seq'::regclass);
+
+
+--
+-- Name: pit_vat id; Type: DEFAULT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_vat ALTER COLUMN id SET DEFAULT nextval('maker.pit_vat_id_seq'::regclass);
+
+
 --
 -- Name: price_feeds id; Type: DEFAULT; Schema: maker; Owner: -
 --
@@ -2017,6 +2267,14 @@ ALTER TABLE ONLY maker.frob
     ADD CONSTRAINT frob_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: pit_drip pit_drip_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_drip
+    ADD CONSTRAINT pit_drip_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: pit_file_debt_ceiling pit_file_debt_ceiling_header_id_tx_idx_log_idx_key; Type: CONSTRAINT; Schema: maker; Owner: -
 --
@@ -2049,6 +2307,46 @@ ALTER TABLE ONLY maker.pit_file_ilk
     ADD CONSTRAINT pit_file_ilk_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: pit_ilk_line pit_ilk_line_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_ilk_line
+    ADD CONSTRAINT pit_ilk_line_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: pit_ilk_spot pit_ilk_spot_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_ilk_spot
+    ADD CONSTRAINT pit_ilk_spot_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: pit_line pit_line_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_line
+    ADD CONSTRAINT pit_line_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: pit_live pit_live_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_live
+    ADD CONSTRAINT pit_live_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: pit_vat pit_vat_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_vat
+    ADD CONSTRAINT pit_vat_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: price_feeds price_feeds_header_id_medianizer_address_tx_idx_log_idx_key; Type: CONSTRAINT; Schema: maker; Owner: -
 --
@@ -2677,4 +2975,5 @@ ALTER TABLE ONLY public.logs
 
 --
 -- PostgreSQL database dump complete
---
+--
+

environments/staging.toml

@@ -11,6 +11,9 @@
 [datadog]
     name = "maker_vdb_staging"
 
+[filesystem]
+    storageDiffsPath = "INSERT-PATH-TO-STORAGE-DIFFS"
+
 [contract]
     [contract.address]
         cat      = "0x2f34f22a00ee4b7a8f8bbc4eaee1658774c624e0"

libraries/shared/event_watcher.go

@@ -25,7 +25,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
-type Watcher struct {
+type EventWatcher struct {
 	Transformers  []shared.Transformer
 	DB            *postgres.DB
 	Fetcher       shared.LogFetcher
@@ -35,10 +35,10 @@ type Watcher struct {
 	StartingBlock *int64
 }
 
-func NewWatcher(db *postgres.DB, bc core.BlockChain) Watcher {
+func NewEventWatcher(db *postgres.DB, bc core.BlockChain) EventWatcher {
 	chunker := shared.NewLogChunker()
 	fetcher := shared.NewFetcher(bc)
-	return Watcher{
+	return EventWatcher{
 		DB:      db,
 		Fetcher: fetcher,
 		Chunker: chunker,
@@ -46,7 +46,7 @@ func NewWatcher(db *postgres.DB, bc core.BlockChain) Watcher {
 }
 
 // Adds transformers to the watcher and updates the chunker, so that it will consider the new transformers.
-func (watcher *Watcher) AddTransformers(initializers []shared.TransformerInitializer) {
+func (watcher *EventWatcher) AddTransformers(initializers []shared.TransformerInitializer) {
 	var contractAddresses []common.Address
 	var topic0s []common.Hash
 	var configs []shared.TransformerConfig
@@ -74,7 +74,7 @@ func (watcher *Watcher) AddTransformers(initializers []shared.TransformerInitial
 	watcher.Chunker.AddConfigs(configs)
 }
 
-func (watcher *Watcher) Execute() error {
+func (watcher *EventWatcher) Execute() error {
 	if watcher.Transformers == nil {
 		return fmt.Errorf("No transformers added to watcher")
 	}

libraries/shared/event_watcher_test.go

@@ -35,12 +35,12 @@ import (
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
-var _ = Describe("Watcher", func() {
+var _ = Describe("EventWatcher", func() {
 	It("initialises correctly", func() {
 		db := test_config.NewTestDB(core.Node{ID: "testNode"})
 		bc := fakes.NewMockBlockChain()
 
-		watcher := shared.NewWatcher(db, bc)
+		watcher := shared.NewEventWatcher(db, bc)
 
 		Expect(watcher.DB).To(Equal(db))
 		Expect(watcher.Fetcher).NotTo(BeNil())
@@ -48,7 +48,7 @@ var _ = Describe("Watcher", func() {
 	})
 
 	It("adds transformers", func() {
-		watcher := shared.NewWatcher(nil, nil)
+		watcher := shared.NewEventWatcher(nil, nil)
 		fakeTransformer := &mocks.MockTransformer{}
 		fakeTransformer.SetTransformerConfig(mocks.FakeTransformerConfig)
 		watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
@@ -60,7 +60,7 @@ var _ = Describe("Watcher", func() {
 	})
 
 	It("adds transformers from multiple sources", func() {
-		watcher := shared.NewWatcher(nil, nil)
+		watcher := shared.NewEventWatcher(nil, nil)
 		fakeTransformer1 := &mocks.MockTransformer{}
 		fakeTransformer1.SetTransformerConfig(mocks.FakeTransformerConfig)
 
@@ -84,7 +84,7 @@ var _ = Describe("Watcher", func() {
 		fakeTransformer2 := &mocks.MockTransformer{}
 		fakeTransformer2.SetTransformerConfig(shared2.TransformerConfig{StartingBlockNumber: 3})
 
-		watcher := shared.NewWatcher(nil, nil)
+		watcher := shared.NewEventWatcher(nil, nil)
 		watcher.AddTransformers([]shared2.TransformerInitializer{
 			fakeTransformer1.FakeTransformerInitializer,
 			fakeTransformer2.FakeTransformerInitializer,
@@ -94,7 +94,7 @@ var _ = Describe("Watcher", func() {
 	})
 
 	It("returns an error when run without transformers", func() {
-		watcher := shared.NewWatcher(nil, nil)
+		watcher := shared.NewEventWatcher(nil, nil)
 		err := watcher.Execute()
 		Expect(err).To(MatchError("No transformers added to watcher"))
 	})
@@ -102,7 +102,7 @@ var _ = Describe("Watcher", func() {
 	Describe("with missing headers", func() {
 		var (
 			db               *postgres.DB
-			watcher          shared.Watcher
+			watcher          shared.EventWatcher
 			mockBlockChain   fakes.MockBlockChain
 			headerRepository repositories.HeaderRepository
 			repository       mocks.MockWatcherRepository
@@ -117,7 +117,7 @@ var _ = Describe("Watcher", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			repository = mocks.MockWatcherRepository{}
-			watcher = shared.NewWatcher(db, &mockBlockChain)
+			watcher = shared.NewEventWatcher(db, &mockBlockChain)
 		})
 
 		It("executes each transformer", func() {
@@ -163,7 +163,7 @@ var _ = Describe("Watcher", func() {
 			mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{logA, logB})
 
 			repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
-			watcher = shared.NewWatcher(db, &mockBlockChain)
+			watcher = shared.NewEventWatcher(db, &mockBlockChain)
 			watcher.AddTransformers([]shared2.TransformerInitializer{
 				transformerA.FakeTransformerInitializer, transformerB.FakeTransformerInitializer})
 

libraries/shared/shared_suite_test.go

@@ -17,8 +17,8 @@
 package shared_test
 
 import (
+	log "github.com/sirupsen/logrus"
 	"io/ioutil"
-	"log"
 	"testing"
 
 	. "github.com/onsi/ginkgo"

libraries/shared/storage_watcher.go

@@ -0,0 +1,74 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package shared
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/sirupsen/logrus"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+	"strings"
+
+	"github.com/vulcanize/vulcanizedb/pkg/fs"
+)
+
+type StorageWatcher struct {
+	db           *postgres.DB
+	tailer       fs.Tailer
+	Transformers map[common.Address]storage.Transformer
+}
+
+func NewStorageWatcher(tailer fs.Tailer, db *postgres.DB) StorageWatcher {
+	transformers := make(map[common.Address]storage.Transformer)
+	return StorageWatcher{
+		db:           db,
+		tailer:       tailer,
+		Transformers: transformers,
+	}
+}
+
+func (watcher StorageWatcher) AddTransformers(initializers []storage.TransformerInitializer) {
+	for _, initializer := range initializers {
+		transformer := initializer(watcher.db)
+		watcher.Transformers[transformer.ContractAddress()] = transformer
+	}
+}
+
+func (watcher StorageWatcher) Execute() error {
+	t, tailErr := watcher.tailer.Tail()
+	if tailErr != nil {
+		return tailErr
+	}
+	for line := range t.Lines {
+		row, parseErr := shared.FromStrings(strings.Split(line.Text, ","))
+		if parseErr != nil {
+			return parseErr
+		}
+		transformer, ok := watcher.Transformers[row.Contract]
+		if !ok {
+			logrus.Warn(shared.ErrContractNotFound{Contract: row.Contract.Hex()}.Error())
+			continue
+		}
+		executeErr := transformer.Execute(row)
+		if executeErr != nil {
+			logrus.Warn(executeErr.Error())
+			continue
+		}
+	}
+	return nil
+}

libraries/shared/storage_watcher_test.go

@@ -0,0 +1,175 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package shared_test
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/hpcloud/tail"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage"
+	shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Storage Watcher", func() {
+	It("adds transformers", func() {
+		fakeAddress := common.HexToAddress("0x12345")
+		fakeTransformer := &mocks.MockStorageTransformer{Address: fakeAddress}
+		watcher := shared.NewStorageWatcher(&fakes.MockTailer{}, test_config.NewTestDB(core.Node{}))
+
+		watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
+
+		Expect(watcher.Transformers[fakeAddress]).To(Equal(fakeTransformer))
+	})
+
+	It("reads the tail of the storage diffs file", func() {
+		mockTailer := fakes.NewMockTailer()
+		watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
+
+		assert(func(err error) {
+			Expect(err).To(BeNil())
+			Expect(mockTailer.TailCalled).To(BeTrue())
+		}, watcher, mockTailer, []*tail.Line{})
+	})
+
+	It("returns error if row parsing fails", func() {
+		mockTailer := fakes.NewMockTailer()
+		watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
+		line := &tail.Line{Text: "oops"}
+
+		assert(func(err error) {
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(shared2.ErrRowMalformed{Length: 1}))
+		}, watcher, mockTailer, []*tail.Line{line})
+	})
+
+	It("logs error if no transformer can parse storage row", func() {
+		mockTailer := fakes.NewMockTailer()
+		line := &tail.Line{
+			Text: "12345,block_hash,123,storage_key,storage_value",
+			Time: time.Time{},
+			Err:  nil,
+		}
+		watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
+		tempFile, err := ioutil.TempFile("", "log")
+		defer os.Remove(tempFile.Name())
+		Expect(err).NotTo(HaveOccurred())
+		logrus.SetOutput(tempFile)
+
+		assert(func(err error) {
+			Expect(err).NotTo(HaveOccurred())
+			logContent, readErr := ioutil.ReadFile(tempFile.Name())
+			Expect(readErr).NotTo(HaveOccurred())
+			Expect(string(logContent)).To(ContainSubstring(shared2.ErrContractNotFound{Contract: common.HexToAddress("0x12345").Hex()}.Error()))
+		}, watcher, mockTailer, []*tail.Line{line})
+	})
+
+	It("executes transformer with storage row", func() {
+		address := []byte{1, 2, 3}
+		blockHash := []byte{4, 5, 6}
+		blockHeight := int64(789)
+		storageKey := []byte{9, 8, 7}
+		storageValue := []byte{6, 5, 4}
+		mockTailer := fakes.NewMockTailer()
+		line := &tail.Line{
+			Text: fmt.Sprintf("%s,%s,%d,%s,%s", common.Bytes2Hex(address), common.Bytes2Hex(blockHash), blockHeight, common.Bytes2Hex(storageKey), common.Bytes2Hex(storageValue)),
+			Time: time.Time{},
+			Err:  nil,
+		}
+		watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
+		fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address)}
+		watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
+
+		assert(func(err error) {
+			Expect(err).To(BeNil())
+			expectedRow, err := shared2.FromStrings(strings.Split(line.Text, ","))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(fakeTransformer.PassedRow).To(Equal(expectedRow))
+		}, watcher, mockTailer, []*tail.Line{line})
+	})
+
+	It("logs error if executing transformer fails", func() {
+		address := []byte{1, 2, 3}
+		blockHash := []byte{4, 5, 6}
+		blockHeight := int64(789)
+		storageKey := []byte{9, 8, 7}
+		storageValue := []byte{6, 5, 4}
+		mockTailer := fakes.NewMockTailer()
+		line := &tail.Line{
+			Text: fmt.Sprintf("%s,%s,%d,%s,%s", common.Bytes2Hex(address), common.Bytes2Hex(blockHash), blockHeight, common.Bytes2Hex(storageKey), common.Bytes2Hex(storageValue)),
+			Time: time.Time{},
+			Err:  nil,
+		}
+		watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
+		executionError := errors.New("storage watcher failed attempting to execute transformer")
+		fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address), ExecuteErr: executionError}
+		watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
+		tempFile, err := ioutil.TempFile("", "log")
+		defer os.Remove(tempFile.Name())
+		Expect(err).NotTo(HaveOccurred())
+		logrus.SetOutput(tempFile)
+
+		assert(func(err error) {
+			Expect(err).NotTo(HaveOccurred())
+			logContent, readErr := ioutil.ReadFile(tempFile.Name())
+			Expect(readErr).NotTo(HaveOccurred())
+			Expect(string(logContent)).To(ContainSubstring(executionError.Error()))
+		}, watcher, mockTailer, []*tail.Line{line})
+	})
+})
+
+func assert(assertion func(err error), watcher shared.StorageWatcher, mockTailer *fakes.MockTailer, lines []*tail.Line) {
+	errs := make(chan error, 1)
+	done := make(chan bool, 1)
+	go execute(watcher, mockTailer, errs, done)
+	for _, line := range lines {
+		mockTailer.Lines <- line
+	}
+	close(mockTailer.Lines)
+
+	select {
+	case err := <-errs:
+		assertion(err)
+		break
+	case <-done:
+		assertion(nil)
+		break
+	}
+}
+
+func execute(watcher shared.StorageWatcher, tailer *fakes.MockTailer, errs chan error, done chan bool) {
+	err := watcher.Execute()
+	if err != nil {
+		errs <- err
+	} else {
+		done <- true
+	}
+}

pkg/datastore/postgres/repositories/header_repository.go

@@ -52,7 +52,7 @@ func (repository HeaderRepository) CreateOrUpdateHeader(header core.Header) (int
 
 func (repository HeaderRepository) GetHeader(blockNumber int64) (core.Header, error) {
 	var header core.Header
-	err := repository.database.Get(&header, `SELECT block_number, hash, raw FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`,
+	err := repository.database.Get(&header, `SELECT id, block_number, hash, raw, block_timestamp FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`,
 		blockNumber, repository.database.Node.ID)
 	return header, err
 }

pkg/datastore/postgres/repositories/header_repository_test.go

@@ -188,9 +188,11 @@ var _ = Describe("Block header repository", func() {
 			dbHeader, err := repo.GetHeader(header.BlockNumber)
 
 			Expect(err).NotTo(HaveOccurred())
+			Expect(dbHeader.Id).NotTo(BeZero())
 			Expect(dbHeader.BlockNumber).To(Equal(header.BlockNumber))
 			Expect(dbHeader.Hash).To(Equal(header.Hash))
 			Expect(dbHeader.Raw).To(MatchJSON(header.Raw))
+			Expect(dbHeader.Timestamp).To(Equal(header.Timestamp))
 		})
 
 		It("does not return header for a different node fingerprint", func() {

pkg/fakes/mock_header_repository.go

@@ -27,8 +27,11 @@ type MockHeaderRepository struct {
 	createOrUpdateHeaderErr                error
 	createOrUpdateHeaderPassedBlockNumbers []int64
 	createOrUpdateHeaderReturnID           int64
+	getHeaderError                         error
+	getHeaderReturnBlockHash               string
 	missingBlockNumbers                    []int64
 	headerExists                           bool
+	GetHeaderPassedBlockNumber             int64
 }
 
 func NewMockHeaderRepository() *MockHeaderRepository {
@@ -53,8 +56,9 @@ func (repository *MockHeaderRepository) CreateOrUpdateHeader(header core.Header)
 	return repository.createOrUpdateHeaderReturnID, repository.createOrUpdateHeaderErr
 }
 
-func (*MockHeaderRepository) GetHeader(blockNumber int64) (core.Header, error) {
+func (repository *MockHeaderRepository) GetHeader(blockNumber int64) (core.Header, error) {
-	return core.Header{BlockNumber: blockNumber}, nil
+	repository.GetHeaderPassedBlockNumber = blockNumber
+	return core.Header{BlockNumber: blockNumber, Hash: repository.getHeaderReturnBlockHash}, repository.getHeaderError
 }
 
 func (repository *MockHeaderRepository) MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) ([]int64, error) {
@@ -69,6 +73,14 @@ func (repository *MockHeaderRepository) SetHeaderExists(headerExists bool) {
 	repository.headerExists = headerExists
 }
 
+func (repository *MockHeaderRepository) SetGetHeaderError(err error) {
+	repository.getHeaderError = err
+}
+
+func (repository *MockHeaderRepository) SetGetHeaderReturnBlockHash(hash string) {
+	repository.getHeaderReturnBlockHash = hash
+}
+
 func (repository *MockHeaderRepository) AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(times int, blockNumbers []int64) {
 	Expect(repository.createOrUpdateHeaderCallCount).To(Equal(times))
 	Expect(repository.createOrUpdateHeaderPassedBlockNumbers).To(Equal(blockNumbers))

pkg/fakes/mock_tailer.go

@@ -0,0 +1,29 @@
+package fakes
+
+import (
+	"github.com/hpcloud/tail"
+	"gopkg.in/tomb.v1"
+)
+
+type MockTailer struct {
+	Lines      chan *tail.Line
+	TailCalled bool
+}
+
+func NewMockTailer() *MockTailer {
+	return &MockTailer{
+		Lines:      make(chan *tail.Line, 1),
+		TailCalled: false,
+	}
+}
+
+func (mock *MockTailer) Tail() (*tail.Tail, error) {
+	mock.TailCalled = true
+	fakeTail := &tail.Tail{
+		Filename: "",
+		Lines:    mock.Lines,
+		Config:   tail.Config{},
+		Tomb:     tomb.Tomb{},
+	}
+	return fakeTail, nil
+}

pkg/fs/reader.go

@@ -22,8 +22,7 @@ type Reader interface {
 	Read(path string) ([]byte, error)
 }
 
-type FsReader struct {
+type FsReader struct{}
-}
 
 func (FsReader) Read(path string) ([]byte, error) {
 	return ioutil.ReadFile(path)

pkg/fs/tail.go

@@ -0,0 +1,15 @@
+package fs
+
+import "github.com/hpcloud/tail"
+
+type Tailer interface {
+	Tail() (*tail.Tail, error)
+}
+
+type FileTailer struct {
+	Path string
+}
+
+func (tailer FileTailer) Tail() (*tail.Tail, error) {
+	return tail.TailFile(tailer.Path, tail.Config{Follow: true})
+}

pkg/transformers/bite/converter.go

@@ -17,7 +17,6 @@
 package bite
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
@@ -64,7 +63,7 @@ func (converter BiteConverter) ToModels(entities []interface{}) ([]interface{},
 			return nil, fmt.Errorf("entity of type %T, not %T", entity, BiteEntity{})
 		}
 
-		ilk := string(bytes.Trim(biteEntity.Ilk[:], "\x00"))
+		ilk := common.Bytes2Hex(biteEntity.Ilk[:])
 		urn := common.BytesToAddress(biteEntity.Urn[:]).String()
 		ink := biteEntity.Ink
 		art := biteEntity.Art

pkg/transformers/bite/converter_test.go

@@ -70,7 +70,7 @@ var _ = Describe("Bite Converter", func() {
 			emptyLog, err := json.Marshal(types.Log{})
 			Expect(err).NotTo(HaveOccurred())
 			expectedModel := bite.BiteModel{
-				Ilk:              "",
+				Ilk:              "0000000000000000000000000000000000000000000000000000000000000000",
 				Urn:              "0x0000000000000000000000000000000000000000",
 				Ink:              "",
 				Art:              "",

pkg/transformers/cat_file/chop_lump/converter.go

@@ -40,7 +40,7 @@ func (CatFileChopLumpConverter) ToModels(ethLogs []types.Log) ([]interface{}, er
 		if err != nil {
 			return nil, err
 		}
-		ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+		ilk := shared.GetHexWithoutPrefix(ethLog.Topics[2].Bytes())
 		what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
 		dataBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:]
 		data := big.NewInt(0).SetBytes(dataBytes).String()

pkg/transformers/cat_file/flip/converter.go

@@ -22,6 +22,7 @@ import (
 	"errors"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
 )
 
@@ -34,7 +35,7 @@ func (CatFileFlipConverter) ToModels(ethLogs []types.Log) ([]interface{}, error)
 		if err != nil {
 			return nil, err
 		}
-		ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+		ilk := shared.GetHexWithoutPrefix(ethLog.Topics[2].Bytes())
 		what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
 		flipBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:]
 		flip := common.BytesToAddress(flipBytes).String()

pkg/transformers/drip_drip/converter.go

@@ -17,10 +17,10 @@
 package drip_drip
 
 import (
-	"bytes"
 	"encoding/json"
 	"errors"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
 type DripDripConverter struct{}
@@ -32,7 +32,7 @@ func (DripDripConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
 		if err != nil {
 			return nil, err
 		}
-		ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+		ilk := shared.GetHexWithoutPrefix(ethLog.Topics[2].Bytes())
 		raw, err := json.Marshal(ethLog)
 		if err != nil {
 			return nil, err

pkg/transformers/drip_file/ilk/converter.go

@@ -38,7 +38,7 @@ func (DripFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error)
 			return nil, err
 		}
 
-		ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+		ilk := shared.GetHexWithoutPrefix(ethLog.Topics[2].Bytes())
 		vow := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
 		taxBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:]
 		tax := shared.ConvertToRay(big.NewInt(0).SetBytes(taxBytes).String())

pkg/transformers/factories/storage/storage_suite_test.go

@@ -0,0 +1,29 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestStorage(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Storage Suite")
+}

pkg/transformers/factories/storage/transformer.go

@@ -0,0 +1,53 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+)
+
+type Transformer struct {
+	Address    common.Address
+	Mappings   storage_diffs.Mappings
+	Repository storage_diffs.Repository
+}
+
+func (transformer Transformer) NewTransformer(db *postgres.DB) storage.Transformer {
+	transformer.Mappings.SetDB(db)
+	transformer.Repository.SetDB(db)
+	return transformer
+}
+
+func (transformer Transformer) ContractAddress() common.Address {
+	return transformer.Address
+}
+
+func (transformer Transformer) Execute(row shared.StorageDiffRow) error {
+	metadata, lookupErr := transformer.Mappings.Lookup(row.StorageKey)
+	if lookupErr != nil {
+		return lookupErr
+	}
+	value, decodeErr := shared.Decode(row, metadata)
+	if decodeErr != nil {
+		return decodeErr
+	}
+	return transformer.Repository.Create(row.BlockHeight, row.BlockHash.Hex(), metadata, value)
+}

pkg/transformers/factories/storage/transformer_test.go

@@ -0,0 +1,102 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/factories/storage"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
+)
+
+var _ = Describe("Storage transformer", func() {
+	var (
+		mappings    *mocks.MockMappings
+		repository  *mocks.MockStorageRepository
+		transformer storage.Transformer
+	)
+
+	BeforeEach(func() {
+		mappings = &mocks.MockMappings{}
+		repository = &mocks.MockStorageRepository{}
+		transformer = storage.Transformer{
+			Address:    common.Address{},
+			Mappings:   mappings,
+			Repository: repository,
+		}
+	})
+
+	It("returns the contract address being watched", func() {
+		fakeAddress := common.HexToAddress("0x12345")
+		transformer.Address = fakeAddress
+
+		Expect(transformer.ContractAddress()).To(Equal(fakeAddress))
+	})
+
+	It("looks up metadata for storage key", func() {
+		transformer.Execute(shared.StorageDiffRow{})
+
+		Expect(mappings.LookupCalled).To(BeTrue())
+	})
+
+	It("returns error if lookup fails", func() {
+		mappings.LookupErr = fakes.FakeError
+
+		err := transformer.Execute(shared.StorageDiffRow{})
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("creates storage row with decoded data", func() {
+		fakeMetadata := shared.StorageValueMetadata{Type: shared.Address}
+		mappings.Metadata = fakeMetadata
+		rawValue := common.HexToAddress("0x12345")
+		fakeBlockNumber := 123
+		fakeBlockHash := "0x67890"
+		fakeRow := shared.StorageDiffRow{
+			Contract:     common.Address{},
+			BlockHash:    common.HexToHash(fakeBlockHash),
+			BlockHeight:  fakeBlockNumber,
+			StorageKey:   common.Hash{},
+			StorageValue: rawValue.Hash(),
+		}
+
+		err := transformer.Execute(fakeRow)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.PassedBlockNumber).To(Equal(fakeBlockNumber))
+		Expect(repository.PassedBlockHash).To(Equal(common.HexToHash(fakeBlockHash).Hex()))
+		Expect(repository.PassedMetadata).To(Equal(fakeMetadata))
+		Expect(repository.PassedValue.(string)).To(Equal(rawValue.Hex()))
+	})
+
+	It("returns error if creating row fails", func() {
+		rawValue := common.HexToAddress("0x12345")
+		fakeMetadata := shared.StorageValueMetadata{Type: shared.Address}
+		mappings.Metadata = fakeMetadata
+		repository.CreateErr = fakes.FakeError
+
+		err := transformer.Execute(shared.StorageDiffRow{StorageValue: rawValue.Hash()})
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+})

pkg/transformers/frob/converter.go

@@ -17,7 +17,6 @@
 package frob
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	log "github.com/sirupsen/logrus"
@@ -68,7 +67,7 @@ func (FrobConverter) ToModels(entities []interface{}) ([]interface{}, error) {
 			return nil, err
 		}
 		model := FrobModel{
-			Ilk:              string(bytes.Trim(frobEntity.Ilk[:], "\x00)")),
+			Ilk:              common.Bytes2Hex(frobEntity.Ilk[:]),
 			Urn:              common.BytesToAddress(frobEntity.Urn[:]).String(),
 			Ink:              frobEntity.Ink.String(),
 			Art:              frobEntity.Art.String(),

pkg/transformers/integration_tests/bite.go

@@ -83,7 +83,7 @@ var _ = Describe("Bite Transformer", func() {
 		Expect(len(dbResult)).To(Equal(1))
 		Expect(dbResult[0].Art).To(Equal("149846666666666655744"))
 		Expect(dbResult[0].IArt).To(Equal("1645356666666666655736"))
-		Expect(dbResult[0].Ilk).To(Equal("ETH"))
+		Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
 		Expect(dbResult[0].Ink).To(Equal("1000000000000000000"))
 		Expect(dbResult[0].NFlip).To(Equal("2"))
 		Expect(dbResult[0].Tab).To(Equal("149846666666666655744"))

pkg/transformers/integration_tests/cat_file.go

@@ -97,12 +97,12 @@ var _ = Describe("Cat File transformer", func() {
 		Expect(len(dbResult)).To(Equal(2))
 		sort.Sort(byLogIndexChopLump(dbResult))
 
-		Expect(dbResult[0].Ilk).To(Equal("REP"))
+		Expect(dbResult[0].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000"))
 		Expect(dbResult[0].What).To(Equal("lump"))
 		Expect(dbResult[0].Data).To(Equal("10000.000000000000000000"))
 		Expect(dbResult[0].LogIndex).To(Equal(uint(3)))
 
-		Expect(dbResult[1].Ilk).To(Equal("REP"))
+		Expect(dbResult[1].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000"))
 		Expect(dbResult[1].What).To(Equal("chop"))
 		Expect(dbResult[1].Data).To(Equal("1.000000000000000000000000000"))
 		Expect(dbResult[1].LogIndex).To(Equal(uint(4)))
@@ -145,7 +145,7 @@ var _ = Describe("Cat File transformer", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(len(dbResult)).To(Equal(1))
-		Expect(dbResult[0].Ilk).To(Equal("ETH"))
+		Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
 		Expect(dbResult[0].What).To(Equal("flip"))
 		Expect(dbResult[0].Flip).To(Equal("0x32D496Ad866D110060866B7125981C73642cc509"))
 	})

pkg/transformers/integration_tests/drip_drip.go

@@ -85,6 +85,6 @@ var _ = Describe("DripDrip Transformer", func() {
 
 		Expect(len(dbResults)).To(Equal(1))
 		dbResult := dbResults[0]
-		Expect(dbResult.Ilk).To(Equal("ETH"))
+		Expect(dbResult.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
 	})
 })

pkg/transformers/integration_tests/frob.go

@@ -94,7 +94,7 @@ var _ = Describe("Frob Transformer", func() {
 		Expect(dbResult[0].Dart).To(Equal("0"))
 		Expect(dbResult[0].Dink).To(Equal("10000000000000"))
 		Expect(dbResult[0].IArt).To(Equal("1495509999999999999992"))
-		Expect(dbResult[0].Ilk).To(Equal("ETH"))
+		Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
 		Expect(dbResult[0].Ink).To(Equal("10050100000000000"))
 		Expect(dbResult[0].Urn).To(Equal("0xc8E093e5f3F9B5Aa6A6b33ea45960b93C161430C"))
 	})

pkg/transformers/integration_tests/pit_file_ilk.go

@@ -87,7 +87,7 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(len(dbResult)).To(Equal(1))
-		Expect(dbResult[0].Ilk).To(Equal("ETH"))
+		Expect(dbResult[0].Ilk).To(Equal("0x4554480000000000000000000000000000000000000000000000000000000000"))
 		Expect(dbResult[0].What).To(Equal("spot"))
 		Expect(dbResult[0].Data).To(Equal("139.840000000000003410605131648"))
 	})
@@ -119,7 +119,7 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() {
 				pitFileIlkLineModel = result
 			}
 		}
-		Expect(pitFileIlkLineModel.Ilk).To(Equal("REP"))
+		Expect(pitFileIlkLineModel.Ilk).To(Equal("0x5245500000000000000000000000000000000000000000000000000000000000"))
 		Expect(pitFileIlkLineModel.Data).To(Equal("2000000.000000000000000000"))
 	})
 })

pkg/transformers/integration_tests/vat_flux.go

@@ -74,7 +74,7 @@ var _ = Describe("VatFlux LogNoteTransformer", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(len(dbResult)).To(Equal(1))
-		Expect(dbResult[0].Ilk).To(Equal("REP"))
+		Expect(dbResult[0].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000"))
 		Expect(dbResult[0].Src).To(Equal("0xC0851F73CC8DD5c0765E71980eC7E7Fd1EF74434"))
 		Expect(dbResult[0].Dst).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
 		Expect(dbResult[0].Rad).To(Equal("1800000000000000000000000000000000000000000000"))

pkg/transformers/integration_tests/vat_fold.go

@@ -82,7 +82,7 @@ var _ = Describe("VatFold Transformer", func() {
 
 		Expect(len(dbResults)).To(Equal(1))
 		dbResult := dbResults[0]
-		Expect(dbResult.Ilk).To(Equal("REP"))
+		Expect(dbResult.Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000"))
 		Expect(dbResult.Urn).To(Equal(common.HexToAddress("0x0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1").String()))
 		Expect(dbResult.Rate).To(Equal("0.000000000000000000000000000"))
 	})

pkg/transformers/integration_tests/vat_grab.go

@@ -75,7 +75,7 @@ var _ = Describe("Vat Grab Transformer", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(len(dbResult)).To(Equal(1))
-		Expect(dbResult[0].Ilk).To(Equal("REP"))
+		Expect(dbResult[0].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000"))
 		Expect(dbResult[0].Urn).To(Equal("0x6a3AE20C315E845B2E398e68EfFe39139eC6060C"))
 		Expect(dbResult[0].V).To(Equal("0x2F34f22a00eE4b7a8F8BBC4eAee1658774c624e0")) //cat contract address
 		Expect(dbResult[0].W).To(Equal("0x3728e9777B2a0a611ee0F89e00E01044ce4736d1"))

pkg/transformers/integration_tests/vat_init.go

@@ -74,6 +74,6 @@ var _ = Describe("VatInit LogNoteTransformer", func() {
 
 		Expect(len(dbResults)).To(Equal(1))
 		dbResult := dbResults[0]
-		Expect(dbResult.Ilk).To(Equal("ETH"))
+		Expect(dbResult.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
 	})
 })

pkg/transformers/integration_tests/vat_slip.go

@@ -66,7 +66,7 @@ var _ = Describe("Vat slip transformer", func() {
 		var model vat_slip.VatSlipModel
 		err = db.Get(&model, `SELECT ilk, guy, rad, tx_idx FROM maker.vat_slip WHERE header_id = $1`, headerID)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(model.Ilk).To(Equal("ETH"))
+		Expect(model.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
 		Expect(model.Guy).To(Equal("0xDA15dCE70ab462E66779f23ee14F21d993789eE3"))
 		Expect(model.Rad).To(Equal("100000000000000000000000000000000000000000000000"))
 		Expect(model.TransactionIndex).To(Equal(uint(0)))

pkg/transformers/integration_tests/vat_tune.go

@@ -75,7 +75,7 @@ var _ = Describe("VatTune LogNoteTransformer", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(len(dbResult)).To(Equal(1))
-		Expect(dbResult[0].Ilk).To(Equal("ETH"))
+		Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
 		Expect(dbResult[0].Urn).To(Equal("0x4F26FfBe5F04ED43630fdC30A87638d53D0b0876"))
 		Expect(dbResult[0].V).To(Equal("0x4F26FfBe5F04ED43630fdC30A87638d53D0b0876"))
 		Expect(dbResult[0].W).To(Equal("0x4F26FfBe5F04ED43630fdC30A87638d53D0b0876"))

pkg/transformers/pit_file/ilk/converter.go

@@ -37,7 +37,7 @@ func (PitFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error)
 		if err != nil {
 			return nil, err
 		}
-		ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+		ilk := ethLog.Topics[2].Hex()
 		what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
 		dataBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:]
 		data, err := getData(dataBytes, what)

pkg/transformers/shared/storage/transformer.go

@@ -0,0 +1,14 @@
+package storage
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+)
+
+type Transformer interface {
+	Execute(row shared.StorageDiffRow) error
+	ContractAddress() common.Address
+}
+
+type TransformerInitializer func(db *postgres.DB) Transformer

pkg/transformers/shared/utilities.go

@@ -17,6 +17,7 @@
 package shared
 
 import (
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
 	"math/big"
 )
@@ -59,6 +60,10 @@ func GetDataBytesAtIndex(n int, logData []byte) []byte {
 	return []byte{}
 }
 
+func GetHexWithoutPrefix(raw []byte) string {
+	return common.Bytes2Hex(raw)
+}
+
 func ConvertToRay(value string) string {
 	return convert(ray, value, rayPrecision)
 }

pkg/transformers/shared/utilities_test.go

@@ -62,4 +62,12 @@ var _ = Describe("Shared utilities", func() {
 			Expect(wadTwo).To(Equal("1.234567890123456690"))
 		})
 	})
+
+	Describe("getting hex without prefix", func() {
+		It("returns bytes as hex without 0x prefix", func() {
+			raw := common.HexToHash("0x4554480000000000000000000000000000000000000000000000000000000000").Bytes()
+			result := shared.GetHexWithoutPrefix(raw)
+			Expect(result).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
+		})
+	})
 })

pkg/transformers/storage_diffs/maker/maker_storage_repository.go

@@ -0,0 +1,38 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package maker
+
+import "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+
+type IMakerStorageRepository interface {
+	GetIlks() ([]string, error)
+	SetDB(db *postgres.DB)
+}
+
+type MakerStorageRepository struct {
+	db *postgres.DB
+}
+
+func (repository *MakerStorageRepository) SetDB(db *postgres.DB) {
+	repository.db = db
+}
+
+func (repository MakerStorageRepository) GetIlks() ([]string, error) {
+	var ilks []string
+	err := repository.db.Select(&ilks, `SELECT DISTINCT ilk FROM maker.vat_init`)
+	return ilks, err
+}

pkg/transformers/storage_diffs/maker/maker_storage_repository_test.go

@@ -0,0 +1,57 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package maker_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Maker storage repository", func() {
+	It("fetches unique ilks from vat init events", func() {
+		db := test_config.NewTestDB(test_config.NewTestNode())
+		test_config.CleanTestDB(db)
+		insertVatInit("ilk1", 1, db)
+		insertVatInit("ilk2", 2, db)
+		insertVatInit("ilk2", 3, db)
+		repository := maker.MakerStorageRepository{}
+		repository.SetDB(db)
+
+		ilks, err := repository.GetIlks()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(ilks)).To(Equal(2))
+		Expect(ilks).To(ConsistOf("ilk1", "ilk2"))
+	})
+})
+
+func insertVatInit(ilk string, blockNumber int64, db *postgres.DB) {
+	headerRepository := repositories.NewHeaderRepository(db)
+	headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(blockNumber))
+	Expect(err).NotTo(HaveOccurred())
+	_, execErr := db.Exec(
+		`INSERT INTO maker.vat_init (header_id, ilk, log_idx, tx_idx, raw_log)
+			VALUES($1, $2, $3, $4, $5)`,
+		headerID, ilk, 0, 0, "[]",
+	)
+	Expect(execErr).NotTo(HaveOccurred())
+}

pkg/transformers/storage_diffs/maker/maker_suite_test.go

@@ -0,0 +1,29 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package maker_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestMaker(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Maker Suite")
+}

pkg/transformers/storage_diffs/maker/pit/mappings.go

@@ -0,0 +1,146 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pit
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+	"math/big"
+)
+
+const (
+	IlkLine = "line"
+	IlkSpot = "spot"
+	PitDrip = "drip"
+	PitLine = "Line"
+	PitLive = "live"
+	PitVat  = "vat"
+)
+
+var (
+	// storage key and value metadata for "drip" on the Pit contract
+	DripKey      = common.HexToHash(storage_diffs.IndexFive)
+	DripMetadata = shared.StorageValueMetadata{
+		Name: PitDrip,
+		Key:  "",
+		Type: shared.Address,
+	}
+
+	IlkSpotIndex = storage_diffs.IndexOne
+
+	// storage key and value metadata for "Spot" on the Pit contract
+	LineKey      = common.HexToHash(storage_diffs.IndexThree)
+	LineMetadata = shared.StorageValueMetadata{
+		Name: PitLine,
+		Key:  "",
+		Type: shared.Uint256,
+	}
+
+	// storage key and value metadata for "live" on the Pit contract
+	LiveKey      = common.HexToHash(storage_diffs.IndexTwo)
+	LiveMetadata = shared.StorageValueMetadata{
+		Name: PitLive,
+		Key:  "",
+		Type: shared.Uint256,
+	}
+
+	// storage key and value metadata for "vat" on the Pit contract
+	VatKey      = common.HexToHash(storage_diffs.IndexFour)
+	VatMetadata = shared.StorageValueMetadata{
+		Name: PitVat,
+		Key:  "",
+		Type: shared.Address,
+	}
+)
+
+type PitMappings struct {
+	StorageRepository maker.IMakerStorageRepository
+	mappings          map[common.Hash]shared.StorageValueMetadata
+}
+
+func (mappings *PitMappings) SetDB(db *postgres.DB) {
+	mappings.StorageRepository.SetDB(db)
+}
+
+func (mappings *PitMappings) Lookup(key common.Hash) (shared.StorageValueMetadata, error) {
+	metadata, ok := mappings.mappings[key]
+	if !ok {
+		err := mappings.loadMappings()
+		if err != nil {
+			return metadata, err
+		}
+		metadata, ok = mappings.mappings[key]
+		if !ok {
+			return metadata, shared.ErrStorageKeyNotFound{Key: key.Hex()}
+		}
+	}
+	return metadata, nil
+}
+
+func (mappings *PitMappings) loadMappings() error {
+	mappings.mappings = getStaticMappings()
+	ilks, err := mappings.StorageRepository.GetIlks()
+	if err != nil {
+		return err
+	}
+	for _, ilk := range ilks {
+		mappings.mappings[getSpotKey(ilk)] = getSpotMetadata(ilk)
+		mappings.mappings[getLineKey(ilk)] = getLineMetadata(ilk)
+	}
+	return nil
+}
+
+func getStaticMappings() map[common.Hash]shared.StorageValueMetadata {
+	mappings := make(map[common.Hash]shared.StorageValueMetadata)
+	mappings[DripKey] = DripMetadata
+	mappings[LineKey] = LineMetadata
+	mappings[LiveKey] = LiveMetadata
+	mappings[VatKey] = VatMetadata
+	return mappings
+}
+
+func getSpotKey(ilk string) common.Hash {
+	keyBytes := common.FromHex("0x" + ilk + IlkSpotIndex)
+	encoded := crypto.Keccak256(keyBytes)
+	return common.BytesToHash(encoded)
+}
+
+func getSpotMetadata(ilk string) shared.StorageValueMetadata {
+	return shared.StorageValueMetadata{
+		Name: IlkSpot,
+		Key:  ilk,
+		Type: shared.Uint256,
+	}
+}
+
+func getLineKey(ilk string) common.Hash {
+	spotMappingAsInt := big.NewInt(0).SetBytes(getSpotKey(ilk).Bytes())
+	incrementedByOne := big.NewInt(0).Add(spotMappingAsInt, big.NewInt(1))
+	return common.BytesToHash(incrementedByOne.Bytes())
+}
+
+func getLineMetadata(ilk string) shared.StorageValueMetadata {
+	return shared.StorageValueMetadata{
+		Name: IlkLine,
+		Key:  ilk,
+		Type: shared.Uint256,
+	}
+}

pkg/transformers/storage_diffs/maker/pit/mappings_test.go

@@ -0,0 +1,90 @@
+package pit_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/pit"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/test_helpers"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+	"math/big"
+)
+
+var _ = Describe("Pit storage mappings", func() {
+	Describe("looking up static keys", func() {
+		It("returns value metadata if key exists", func() {
+			storageRepository := &test_helpers.MockMakerStorageRepository{}
+			mappings := pit.PitMappings{StorageRepository: storageRepository}
+
+			Expect(mappings.Lookup(pit.DripKey)).To(Equal(pit.DripMetadata))
+			Expect(mappings.Lookup(pit.LineKey)).To(Equal(pit.LineMetadata))
+			Expect(mappings.Lookup(pit.LiveKey)).To(Equal(pit.LiveMetadata))
+			Expect(mappings.Lookup(pit.VatKey)).To(Equal(pit.VatMetadata))
+		})
+
+		It("returns error if key does not exist", func() {
+			mappings := pit.PitMappings{StorageRepository: &test_helpers.MockMakerStorageRepository{}}
+
+			_, err := mappings.Lookup(common.HexToHash(fakes.FakeHash.Hex()))
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(shared.ErrStorageKeyNotFound{Key: fakes.FakeHash.Hex()}))
+		})
+	})
+
+	Describe("looking up dynamic keys", func() {
+		It("refreshes mappings from repository if key not found", func() {
+			storageRepository := &test_helpers.MockMakerStorageRepository{}
+			mappings := pit.PitMappings{StorageRepository: storageRepository}
+
+			mappings.Lookup(fakes.FakeHash)
+
+			Expect(storageRepository.GetIlksCalled).To(BeTrue())
+		})
+
+		It("returns value metadata for spot when ilk in the DB", func() {
+			storageRepository := &test_helpers.MockMakerStorageRepository{}
+			fakeIlk := "fakeIlk"
+			storageRepository.SetIlks([]string{fakeIlk})
+			mappings := pit.PitMappings{StorageRepository: storageRepository}
+			ilkSpotKey := common.BytesToHash(crypto.Keccak256(common.FromHex("0x" + fakeIlk + pit.IlkSpotIndex)))
+			expectedMetadata := shared.StorageValueMetadata{
+				Name: pit.IlkSpot,
+				Key:  fakeIlk,
+				Type: shared.Uint256,
+			}
+
+			Expect(mappings.Lookup(ilkSpotKey)).To(Equal(expectedMetadata))
+		})
+
+		It("returns value metadata for line when ilk in the DB", func() {
+			storageRepository := &test_helpers.MockMakerStorageRepository{}
+			fakeIlk := "fakeIlk"
+			storageRepository.SetIlks([]string{fakeIlk})
+			mappings := pit.PitMappings{StorageRepository: storageRepository}
+			ilkSpotKeyBytes := crypto.Keccak256(common.FromHex("0x" + fakeIlk + pit.IlkSpotIndex))
+			ilkSpotAsInt := big.NewInt(0).SetBytes(ilkSpotKeyBytes)
+			incrementedIlkSpot := big.NewInt(0).Add(ilkSpotAsInt, big.NewInt(1))
+			ilkLineKey := common.BytesToHash(incrementedIlkSpot.Bytes())
+			expectedMetadata := shared.StorageValueMetadata{
+				Name: pit.IlkLine,
+				Key:  fakeIlk,
+				Type: shared.Uint256,
+			}
+
+			Expect(mappings.Lookup(ilkLineKey)).To(Equal(expectedMetadata))
+		})
+
+		It("returns error if key not found", func() {
+			storageRepository := &test_helpers.MockMakerStorageRepository{}
+			mappings := pit.PitMappings{StorageRepository: storageRepository}
+
+			_, err := mappings.Lookup(fakes.FakeHash)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(shared.ErrStorageKeyNotFound{Key: fakes.FakeHash.Hex()}))
+		})
+	})
+})

pkg/transformers/storage_diffs/maker/pit/pit_suite_test.go

@@ -0,0 +1,13 @@
+package pit_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestPit(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Pit Suite")
+}

pkg/transformers/storage_diffs/maker/pit/repository.go

@@ -0,0 +1,79 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pit
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+)
+
+type PitStorageRepository struct {
+	db *postgres.DB
+}
+
+func (repository *PitStorageRepository) SetDB(db *postgres.DB) {
+	repository.db = db
+}
+
+func (repository PitStorageRepository) Create(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, value interface{}) error {
+	switch metadata.Name {
+	case IlkLine:
+		return repository.insertIlkLine(blockNumber, blockHash, metadata.Key, value.(string))
+	case IlkSpot:
+		return repository.insertIlkSpot(blockNumber, blockHash, metadata.Key, value.(string))
+	case PitDrip:
+		return repository.insertPitDrip(blockNumber, blockHash, value.(string))
+	case PitLine:
+		return repository.insertPitLine(blockNumber, blockHash, value.(string))
+	case PitLive:
+		return repository.insertPitLive(blockNumber, blockHash, value.(string))
+	case PitVat:
+		return repository.insertPitVat(blockNumber, blockHash, value.(string))
+	default:
+		panic("unrecognized storage metadata name")
+	}
+}
+
+func (repository PitStorageRepository) insertIlkLine(blockNumber int, blockHash string, ilk string, line string) error {
+	_, err := repository.db.Exec(`INSERT INTO maker.pit_ilk_line (block_number, block_hash, ilk, line) VALUES ($1, $2, $3, $4)`, blockNumber, blockHash, ilk, line)
+	return err
+}
+
+func (repository PitStorageRepository) insertIlkSpot(blockNumber int, blockHash string, ilk string, spot string) error {
+	_, err := repository.db.Exec(`INSERT INTO maker.pit_ilk_spot (block_number, block_hash, ilk, spot) VALUES ($1, $2, $3, $4)`, blockNumber, blockHash, ilk, spot)
+	return err
+}
+
+func (repository PitStorageRepository) insertPitDrip(blockNumber int, blockHash string, drip string) error {
+	_, err := repository.db.Exec(`INSERT INTO maker.pit_drip (block_number, block_hash, drip) VALUES ($1, $2, $3)`, blockNumber, blockHash, drip)
+	return err
+}
+
+func (repository PitStorageRepository) insertPitLine(blockNumber int, blockHash string, line string) error {
+	_, err := repository.db.Exec(`INSERT INTO maker.pit_line (block_number, block_hash, line) VALUES ($1, $2, $3)`, blockNumber, blockHash, line)
+	return err
+}
+
+func (repository PitStorageRepository) insertPitLive(blockNumber int, blockHash string, live string) error {
+	_, err := repository.db.Exec(`INSERT INTO maker.pit_live (block_number, block_hash, live) VALUES ($1, $2, $3)`, blockNumber, blockHash, live)
+	return err
+}
+
+func (repository PitStorageRepository) insertPitVat(blockNumber int, blockHash string, vat string) error {
+	_, err := repository.db.Exec(`INSERT INTO maker.pit_vat (block_number, block_hash, vat) VALUES ($1, $2, $3)`, blockNumber, blockHash, vat)
+	return err
+}

pkg/transformers/storage_diffs/maker/pit/repository_test.go

@@ -0,0 +1,172 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pit_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/pit"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Pit storage repository", func() {
+	var (
+		blockNumber int
+		blockHash   string
+		db          *postgres.DB
+		err         error
+		repo        pit.PitStorageRepository
+	)
+
+	BeforeEach(func() {
+		blockNumber = 123
+		blockHash = "expected_block_hash"
+		db = test_config.NewTestDB(test_config.NewTestNode())
+		test_config.CleanTestDB(db)
+		repo = pit.PitStorageRepository{}
+		repo.SetDB(db)
+	})
+
+	It("persists an ilk line", func() {
+		expectedIlk := "fake_ilk"
+		expectedLine := "12345"
+		ilkLineMetadata := shared.StorageValueMetadata{
+			Name: pit.IlkLine,
+			Key:  expectedIlk,
+			Type: shared.Uint256,
+		}
+		err = repo.Create(blockNumber, blockHash, ilkLineMetadata, expectedLine)
+
+		Expect(err).NotTo(HaveOccurred())
+		type IlkLine struct {
+			BlockMetadata
+			Ilk  string
+			Line string
+		}
+		var result IlkLine
+		err = db.Get(&result, `SELECT block_number, block_hash, ilk, line FROM maker.pit_ilk_line`)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.BlockNumber).To(Equal(blockNumber))
+		Expect(result.BlockHash).To(Equal(blockHash))
+		Expect(result.Ilk).To(Equal(expectedIlk))
+		Expect(result.Line).To(Equal(expectedLine))
+	})
+
+	It("persists an ilk spot", func() {
+		expectedIlk := "fake_ilk"
+		expectedSpot := "12345"
+		ilkSpotMetadata := shared.StorageValueMetadata{
+			Name: pit.IlkSpot,
+			Key:  expectedIlk,
+			Type: shared.Uint256,
+		}
+		err = repo.Create(blockNumber, blockHash, ilkSpotMetadata, expectedSpot)
+
+		Expect(err).NotTo(HaveOccurred())
+		type IlkSpot struct {
+			BlockMetadata
+			Ilk  string
+			Spot string
+		}
+		var result IlkSpot
+		err = db.Get(&result, `SELECT block_number, block_hash, ilk, spot FROM maker.pit_ilk_spot`)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.BlockNumber).To(Equal(blockNumber))
+		Expect(result.BlockHash).To(Equal(blockHash))
+		Expect(result.Ilk).To(Equal(expectedIlk))
+		Expect(result.Spot).To(Equal(expectedSpot))
+	})
+
+	It("persists a pit drip", func() {
+		expectedDrip := "0x0123456789abcdef0123"
+
+		err = repo.Create(blockNumber, blockHash, pit.DripMetadata, expectedDrip)
+
+		Expect(err).NotTo(HaveOccurred())
+		type PitDrip struct {
+			BlockMetadata
+			Drip string
+		}
+		var result PitDrip
+		err = db.Get(&result, `SELECT block_number, block_hash, drip FROM maker.pit_drip`)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.BlockNumber).To(Equal(blockNumber))
+		Expect(result.BlockHash).To(Equal(blockHash))
+		Expect(result.Drip).To(Equal(expectedDrip))
+	})
+
+	It("persists a pit line", func() {
+		expectedLine := "12345"
+
+		err = repo.Create(blockNumber, blockHash, pit.LineMetadata, expectedLine)
+
+		Expect(err).NotTo(HaveOccurred())
+		type PitLine struct {
+			BlockMetadata
+			Line string
+		}
+		var result PitLine
+		err = db.Get(&result, `SELECT block_number, block_hash, line FROM maker.pit_line`)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.BlockNumber).To(Equal(blockNumber))
+		Expect(result.BlockHash).To(Equal(blockHash))
+		Expect(result.Line).To(Equal(expectedLine))
+	})
+
+	It("persists a pit live", func() {
+		expectedLive := "12345"
+
+		err = repo.Create(blockNumber, blockHash, pit.LiveMetadata, expectedLive)
+
+		Expect(err).NotTo(HaveOccurred())
+		type PitLive struct {
+			BlockMetadata
+			Live string
+		}
+		var result PitLive
+		err = db.Get(&result, `SELECT block_number, block_hash, live FROM maker.pit_live`)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.BlockNumber).To(Equal(blockNumber))
+		Expect(result.BlockHash).To(Equal(blockHash))
+		Expect(result.Live).To(Equal(expectedLive))
+	})
+
+	It("persists a pit vat", func() {
+		expectedVat := "0x0123456789abcdef0123"
+
+		err = repo.Create(blockNumber, blockHash, pit.VatMetadata, expectedVat)
+
+		Expect(err).NotTo(HaveOccurred())
+		type PitVat struct {
+			BlockMetadata
+			Vat string
+		}
+		var result PitVat
+		err = db.Get(&result, `SELECT block_number, block_hash, vat FROM maker.pit_vat`)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.BlockNumber).To(Equal(blockNumber))
+		Expect(result.BlockHash).To(Equal(blockHash))
+		Expect(result.Vat).To(Equal(expectedVat))
+	})
+})
+
+type BlockMetadata struct {
+	BlockNumber int    `db:"block_number"`
+	BlockHash   string `db:"block_hash"`
+}

pkg/transformers/storage_diffs/maker/test_helpers/maker_storage_repository.go

@@ -0,0 +1,19 @@
+package test_helpers
+
+import "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+
+type MockMakerStorageRepository struct {
+	GetIlksCalled bool
+	ilks          []string
+}
+
+func (repository *MockMakerStorageRepository) GetIlks() ([]string, error) {
+	repository.GetIlksCalled = true
+	return repository.ilks, nil
+}
+
+func (repository *MockMakerStorageRepository) SetDB(db *postgres.DB) {}
+
+func (repository *MockMakerStorageRepository) SetIlks(ilks []string) {
+	repository.ilks = ilks
+}

pkg/transformers/storage_diffs/mappings.go

@@ -0,0 +1,39 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage_diffs
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+)
+
+type Mappings interface {
+	Lookup(key common.Hash) (shared.StorageValueMetadata, error)
+	SetDB(db *postgres.DB)
+}
+
+const (
+	IndexZero  = "0000000000000000000000000000000000000000000000000000000000000000"
+	IndexOne   = "0000000000000000000000000000000000000000000000000000000000000001"
+	IndexTwo   = "0000000000000000000000000000000000000000000000000000000000000002"
+	IndexThree = "0000000000000000000000000000000000000000000000000000000000000003"
+	IndexFour  = "0000000000000000000000000000000000000000000000000000000000000004"
+	IndexFive  = "0000000000000000000000000000000000000000000000000000000000000005"
+	IndexSix   = "0000000000000000000000000000000000000000000000000000000000000006"
+	IndexSeven = "0000000000000000000000000000000000000000000000000000000000000007"
+)

pkg/transformers/storage_diffs/repository.go

@@ -0,0 +1,27 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage_diffs
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+)
+
+type Repository interface {
+	Create(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, value interface{}) error
+	SetDB(db *postgres.DB)
+}

pkg/transformers/storage_diffs/shared/decoder.go

@@ -0,0 +1,42 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package shared
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"math/big"
+)
+
+func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, error) {
+	switch metadata.Type {
+	case Uint256:
+		return decodeUint256(row.StorageValue.Bytes()), nil
+	case Address:
+		return decodeAddress(row.StorageValue.Bytes()), nil
+	default:
+		return nil, ErrTypeNotFound{}
+	}
+}
+
+func decodeUint256(raw []byte) string {
+	n := big.NewInt(0).SetBytes(raw)
+	return n.String()
+}
+
+func decodeAddress(raw []byte) string {
+	return common.BytesToAddress(raw).Hex()
+}

pkg/transformers/storage_diffs/shared/decoder_test.go

@@ -0,0 +1,58 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package shared_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+	"math/big"
+)
+
+var _ = Describe("Storage decoder", func() {
+	It("decodes uint256", func() {
+		fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000539")
+		row := shared.StorageDiffRow{StorageValue: fakeInt}
+		metadata := shared.StorageValueMetadata{Type: shared.Uint256}
+
+		result, err := shared.Decode(row, metadata)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
+	})
+
+	It("decodes address", func() {
+		fakeAddress := common.HexToAddress("0x12345")
+		row := shared.StorageDiffRow{StorageValue: fakeAddress.Hash()}
+		metadata := shared.StorageValueMetadata{Type: shared.Address}
+
+		result, err := shared.Decode(row, metadata)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result).To(Equal(fakeAddress.Hex()))
+	})
+
+	It("returns error if attempting to decode unknown type", func() {
+		metadata := shared.StorageValueMetadata{Type: 100}
+
+		_, err := shared.Decode(shared.StorageDiffRow{}, metadata)
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(shared.ErrTypeNotFound{}))
+	})
+})

pkg/transformers/storage_diffs/shared/errors.go

@@ -0,0 +1,61 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package shared
+
+import "fmt"
+
+type ErrContractNotFound struct {
+	Contract string
+}
+
+func (e ErrContractNotFound) Error() string {
+	return fmt.Sprintf("transformer not found for contract: %s", e.Contract)
+}
+
+type ErrHeaderMismatch struct {
+	BlockHeight int
+	DbHash      string
+	DiffHash    string
+}
+
+func (e ErrHeaderMismatch) Error() string {
+	return fmt.Sprintf("header hash in row does not match db at height %d - row: %s, db: %s", e.BlockHeight, e.DbHash, e.DiffHash)
+}
+
+type ErrRowMalformed struct {
+	Length int
+}
+
+func (e ErrRowMalformed) Error() string {
+	return fmt.Sprintf("storage row malformed: length %d, expected %d", e.Length, ExpectedRowLength)
+}
+
+type ErrStorageKeyNotFound struct {
+	Key string
+}
+
+func (e ErrStorageKeyNotFound) Error() string {
+	return fmt.Sprintf("unknown storage key: %s", e.Key)
+}
+
+type ErrTypeNotFound struct {
+	Type int
+}
+
+func (e ErrTypeNotFound) Error() string {
+	return fmt.Sprintf("no decoder for type: %d", e.Type)
+}

pkg/transformers/storage_diffs/shared/row.go

@@ -0,0 +1,49 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package shared
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"strconv"
+)
+
+const ExpectedRowLength = 5
+
+type StorageDiffRow struct {
+	Contract     common.Address
+	BlockHash    common.Hash
+	BlockHeight  int
+	StorageKey   common.Hash
+	StorageValue common.Hash
+}
+
+func FromStrings(csvRow []string) (StorageDiffRow, error) {
+	if len(csvRow) != ExpectedRowLength {
+		return StorageDiffRow{}, ErrRowMalformed{Length: len(csvRow)}
+	}
+	height, err := strconv.Atoi(csvRow[2])
+	if err != nil {
+		return StorageDiffRow{}, err
+	}
+	return StorageDiffRow{
+		Contract:     common.HexToAddress(csvRow[0]),
+		BlockHash:    common.HexToHash(csvRow[1]),
+		BlockHeight:  height,
+		StorageKey:   common.HexToHash(csvRow[3]),
+		StorageValue: common.HexToHash(csvRow[4]),
+	}, nil
+}

pkg/transformers/storage_diffs/shared/row_test.go

@@ -0,0 +1,57 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package shared_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+)
+
+var _ = Describe("Storage row parsing", func() {
+	It("converts an array of strings to a row struct", func() {
+		contract := "0x123"
+		blockHash := "0x456"
+		blockHeight := "789"
+		storageKey := "0x987"
+		storageValue := "0x654"
+		data := []string{contract, blockHash, blockHeight, storageKey, storageValue}
+
+		result, err := shared.FromStrings(data)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.Contract).To(Equal(common.HexToAddress(contract)))
+		Expect(result.BlockHash).To(Equal(common.HexToHash(blockHash)))
+		Expect(result.BlockHeight).To(Equal(789))
+		Expect(result.StorageKey).To(Equal(common.HexToHash(storageKey)))
+		Expect(result.StorageValue).To(Equal(common.HexToHash(storageValue)))
+	})
+
+	It("returns an error if row is missing data", func() {
+		_, err := shared.FromStrings([]string{"0x123"})
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(shared.ErrRowMalformed{Length: 1}))
+	})
+
+	It("returns error if block height malformed", func() {
+		_, err := shared.FromStrings([]string{"", "", "", "", ""})
+
+		Expect(err).To(HaveOccurred())
+	})
+})

pkg/transformers/storage_diffs/shared/shared_suite_test.go

@@ -0,0 +1,29 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package shared_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestShared(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Shared Suite")
+}

pkg/transformers/storage_diffs/shared/value.go

@@ -0,0 +1,31 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package shared
+
+type ValueType int
+
+const (
+	Uint256 ValueType = iota
+	Bytes32
+	Address
+)
+
+type StorageValueMetadata struct {
+	Name string
+	Key  string
+	Type ValueType
+}

pkg/transformers/storage_transformers.go

@@ -0,0 +1,17 @@
+package transformers
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/factories/storage"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/pit"
+)
+
+func GetPitStorageTransformer() storage.Transformer {
+	return storage.Transformer{
+		Address:    common.HexToAddress(constants.PitContractAddress()),
+		Mappings:   &pit.PitMappings{StorageRepository: &maker.MakerStorageRepository{}},
+		Repository: &pit.PitStorageRepository{},
+	}
+}

pkg/transformers/test_data/bite.go

@@ -41,7 +41,7 @@ var (
 	biteRawJson, _ = json.Marshal(EthBiteLog)
 	biteIlk        = [32]byte{69, 84, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
 	biteLad        = [32]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 180, 20, 126, 218, 128, 254, 199, 18, 42, 225, 109, 162, 71, 156, 189, 127, 251}
-	biteIlkString  = "ETH"
+	biteIlkString  = "4554480000000000000000000000000000000000000000000000000000000000"
 	biteLadString  = "0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"
 )
 

pkg/transformers/test_data/cat_file.go

@@ -45,7 +45,7 @@ var EthCatFileChopLog = types.Log{
 }
 var rawCatFileChopLog, _ = json.Marshal(EthCatFileChopLog)
 var CatFileChopModel = chop_lump.CatFileChopLumpModel{
-	Ilk:              "fake ilk",
+	Ilk:              "66616b6520696c6b000000000000000000000000000000000000000000000000",
 	What:             "chop",
 	Data:             "123.456789012345680589533003513",
 	TransactionIndex: EthCatFileChopLog.TxIndex,
@@ -71,7 +71,7 @@ var EthCatFileLumpLog = types.Log{
 }
 var rawCatFileLumpLog, _ = json.Marshal(EthCatFileLumpLog)
 var CatFileLumpModel = chop_lump.CatFileChopLumpModel{
-	Ilk:              "fake ilk",
+	Ilk:              "66616b6520696c6b000000000000000000000000000000000000000000000000",
 	What:             "lump",
 	Data:             "12345.678901234567092615",
 	TransactionIndex: EthCatFileLumpLog.TxIndex,
@@ -98,7 +98,7 @@ var EthCatFileFlipLog = types.Log{
 
 var rawCatFileFlipLog, _ = json.Marshal(EthCatFileFlipLog)
 var CatFileFlipModel = flip.CatFileFlipModel{
-	Ilk:              "fake ilk",
+	Ilk:              "66616b6520696c6b000000000000000000000000000000000000000000000000",
 	What:             "flip",
 	Flip:             "0x07Fa9eF6609cA7921112231F8f195138ebbA2977",
 	TransactionIndex: EthCatFileFlipLog.TxIndex,

pkg/transformers/test_data/drip_drip.go

@@ -44,7 +44,7 @@ var EthDripDripLog = types.Log{
 
 var rawDripDripLog, _ = json.Marshal(EthDripDripLog)
 var DripDripModel = drip_drip.DripDripModel{
-	Ilk:              "fake ilk",
+	Ilk:              "66616b6520696c6b000000000000000000000000000000000000000000000000",
 	LogIndex:         EthDripDripLog.Index,
 	TransactionIndex: EthDripDripLog.TxIndex,
 	Raw:              rawDripDripLog,

pkg/transformers/test_data/drip_file.go

@@ -47,7 +47,7 @@ var EthDripFileIlkLog = types.Log{
 
 var rawDripFileIlkLog, _ = json.Marshal(EthDripFileIlkLog)
 var DripFileIlkModel = ilk2.DripFileIlkModel{
-	Ilk:              "fake ilk",
+	Ilk:              "66616b6520696c6b000000000000000000000000000000000000000000000000",
 	Vow:              "fake vow",
 	Tax:              "12300.000000000000000000000000000",
 	LogIndex:         EthDripFileIlkLog.Index,

pkg/transformers/test_data/frob.go

@@ -41,7 +41,7 @@ var (
 	frobLad       = [32]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 217, 34, 137, 65, 83, 190, 158, 239, 123, 114, 24, 220, 86, 93, 29, 12, 226, 160, 146}
 	ink           = big.NewInt(15)
 	ilk           = [32]byte{102, 97, 107, 101, 32, 105, 108, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
-	frobIlkString = "fake ilk"
+	frobIlkString = "66616b6520696c6b000000000000000000000000000000000000000000000000"
 	frobUrnString = "0x64d922894153BE9EEf7b7218dc565d1D0Ce2a092"
 )
 

pkg/transformers/test_data/mocks/mappings.go

@@ -0,0 +1,22 @@
+package mocks
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+)
+
+type MockMappings struct {
+	Metadata     shared.StorageValueMetadata
+	LookupCalled bool
+	LookupErr    error
+}
+
+func (mappings *MockMappings) Lookup(key common.Hash) (shared.StorageValueMetadata, error) {
+	mappings.LookupCalled = true
+	return mappings.Metadata, mappings.LookupErr
+}
+
+func (*MockMappings) SetDB(db *postgres.DB) {
+	panic("implement me")
+}

pkg/transformers/test_data/mocks/storage_repository.go

@@ -0,0 +1,26 @@
+package mocks
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+)
+
+type MockStorageRepository struct {
+	CreateErr         error
+	PassedBlockNumber int
+	PassedBlockHash   string
+	PassedMetadata    shared.StorageValueMetadata
+	PassedValue       interface{}
+}
+
+func (repository *MockStorageRepository) Create(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, value interface{}) error {
+	repository.PassedBlockNumber = blockNumber
+	repository.PassedBlockHash = blockHash
+	repository.PassedMetadata = metadata
+	repository.PassedValue = value
+	return repository.CreateErr
+}
+
+func (*MockStorageRepository) SetDB(db *postgres.DB) {
+	panic("implement me")
+}

pkg/transformers/test_data/mocks/storage_transformer.go

@@ -0,0 +1,27 @@
+package mocks
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+)
+
+type MockStorageTransformer struct {
+	Address    common.Address
+	ExecuteErr error
+	PassedRow  shared.StorageDiffRow
+}
+
+func (transformer *MockStorageTransformer) Execute(row shared.StorageDiffRow) error {
+	transformer.PassedRow = row
+	return transformer.ExecuteErr
+}
+
+func (transformer *MockStorageTransformer) ContractAddress() common.Address {
+	return transformer.Address
+}
+
+func (transformer *MockStorageTransformer) FakeTransformerInitializer(db *postgres.DB) storage.Transformer {
+	return transformer
+}

pkg/transformers/test_data/pit_file.go

@@ -75,7 +75,7 @@ var EthPitFileIlkLineLog = types.Log{
 
 var rawPitFileIlkLineLog, _ = json.Marshal(EthPitFileIlkLineLog)
 var PitFileIlkLineModel = ilk2.PitFileIlkModel{
-	Ilk:              "fake ilk",
+	Ilk:              "0x66616b6520696c6b000000000000000000000000000000000000000000000000",
 	What:             "line",
 	Data:             "0.000001000000000000",
 	LogIndex:         EthPitFileIlkLineLog.Index,
@@ -102,7 +102,7 @@ var EthPitFileIlkSpotLog = types.Log{
 
 var rawPitFileIlkSpotLog, _ = json.Marshal(EthPitFileIlkSpotLog)
 var PitFileIlkSpotModel = ilk2.PitFileIlkModel{
-	Ilk:              "fake ilk",
+	Ilk:              "0x66616b6520696c6b000000000000000000000000000000000000000000000000",
 	What:             "spot",
 	Data:             "0.000000000000001000000000000",
 	LogIndex:         EthPitFileIlkSpotLog.Index,

pkg/transformers/test_data/vat_flux.go

@@ -44,7 +44,7 @@ var VatFluxLog = types.Log{
 
 var rawFluxLog, _ = json.Marshal(VatFluxLog)
 var VatFluxModel = vat_flux.VatFluxModel{
-	Ilk:              "REP",
+	Ilk:              "5245500000000000000000000000000000000000000000000000000000000000",
 	Src:              "0x7FA9EF6609Ca7921112231f8f195138ebba29770",
 	Dst:              "0x93086347c52a8878af71bB818509d484c6a2e1bF",
 	Rad:              "123",

pkg/transformers/test_data/vat_fold.go

@@ -46,7 +46,7 @@ var EthVatFoldLog = types.Log{
 
 var rawVatFoldLog, _ = json.Marshal(EthVatFoldLog)
 var VatFoldModel = vat_fold.VatFoldModel{
-	Ilk:              "REP",
+	Ilk:              "5245500000000000000000000000000000000000000000000000000000000000",
 	Urn:              "0x3728e9777B2a0a611ee0F89e00E01044ce4736d1",
 	Rate:             "0.000000000000000000000000002",
 	LogIndex:         EthVatFoldLog.Index,

pkg/transformers/test_data/vat_grab.go

@@ -28,7 +28,7 @@ var EthVatGrabLog = types.Log{
 
 var rawVatGrabLog, _ = json.Marshal(EthVatGrabLog)
 var VatGrabModel = vat_grab.VatGrabModel{
-	Ilk:              "REP",
+	Ilk:              "5245500000000000000000000000000000000000000000000000000000000000",
 	Urn:              "0x6a3AE20C315E845B2E398e68EfFe39139eC6060C",
 	V:                "0x2F34f22a00eE4b7a8F8BBC4eAee1658774c624e0",
 	W:                "0x3728e9777B2a0a611ee0F89e00E01044ce4736d1",

pkg/transformers/test_data/vat_init.go

@@ -46,7 +46,7 @@ var EthVatInitLog = types.Log{
 
 var rawVatInitLog, _ = json.Marshal(EthVatInitLog)
 var VatInitModel = vat_init.VatInitModel{
-	Ilk:              "fake ilk",
+	Ilk:              "66616b6520696c6b000000000000000000000000000000000000000000000000",
 	LogIndex:         EthVatInitLog.Index,
 	TransactionIndex: EthVatInitLog.TxIndex,
 	Raw:              rawVatInitLog,

pkg/transformers/test_data/vat_slip.go

@@ -46,7 +46,7 @@ var EthVatSlipLog = types.Log{
 
 var rawVatSlipLog, _ = json.Marshal(EthVatSlipLog)
 var VatSlipModel = vat_slip.VatSlipModel{
-	Ilk:              "fake ilk",
+	Ilk:              "66616b6520696c6b000000000000000000000000000000000000000000000000",
 	Guy:              common.HexToAddress("0x7d7bEe5fCfD8028cf7b00876C5b1421c800561A6").String(),
 	Rad:              "987654321",
 	TransactionIndex: EthVatSlipLog.TxIndex,

pkg/transformers/test_data/vat_toll.go

@@ -29,7 +29,7 @@ var EthVatTollLog = types.Log{
 
 var rawVatTollLog, _ = json.Marshal(EthVatTollLog)
 var VatTollModel = vat_toll.VatTollModel{
-	Ilk:              "fake ilk",
+	Ilk:              "66616b6520696c6b000000000000000000000000000000000000000000000000",
 	Urn:              "0xA3E37186E017747DbA34042e83e3F76Ad3CcE9b0",
 	Take:             big.NewInt(123456789).String(),
 	TransactionIndex: EthVatTollLog.TxIndex,

pkg/transformers/test_data/vat_tune.go

@@ -34,7 +34,7 @@ var dartString = "11579208923731619542357098500868790785326998466564056403945558
 var vatTuneDart, _ = new(big.Int).SetString(dartString, 10)
 var urnAddress = "0x4F26FfBe5F04ED43630fdC30A87638d53D0b0876"
 var VatTuneModel = vat_tune.VatTuneModel{
-	Ilk:              "ETH",
+	Ilk:              "4554480000000000000000000000000000000000000000000000000000000000",
 	Urn:              urnAddress,
 	V:                urnAddress,
 	W:                urnAddress,

pkg/transformers/vat_flux/converter.go

@@ -17,7 +17,6 @@
 package vat_flux
 
 import (
-	"bytes"
 	"encoding/json"
 	"errors"
 	"github.com/ethereum/go-ethereum/common"
@@ -36,7 +35,7 @@ func (VatFluxConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
 			return nil, err
 		}
 
-		ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
+		ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes())
 		src := common.BytesToAddress(ethLog.Topics[2].Bytes())
 		dst := common.BytesToAddress(ethLog.Topics[3].Bytes())
 		radBytes := shared.GetDataBytesAtIndex(-1, ethLog.Data)

pkg/transformers/vat_fold/converter.go

@@ -17,7 +17,6 @@
 package vat_fold
 
 import (
-	"bytes"
 	"encoding/json"
 	"errors"
 	"github.com/ethereum/go-ethereum/common"
@@ -35,7 +34,7 @@ func (VatFoldConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
 			return nil, err
 		}
 
-		ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
+		ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes())
 		urn := common.BytesToAddress(ethLog.Topics[2].Bytes()).String()
 		rate := shared.ConvertToRay(ethLog.Topics[3].Big().String())
 		raw, err := json.Marshal(ethLog)

pkg/transformers/vat_grab/converter.go

@@ -1,7 +1,6 @@
 package vat_grab
 
 import (
-	"bytes"
 	"encoding/json"
 	"errors"
 	"github.com/ethereum/go-ethereum/common"
@@ -20,7 +19,7 @@ func (VatGrabConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
 		if err != nil {
 			return nil, err
 		}
-		ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
+		ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes())
 		urn := common.BytesToAddress(ethLog.Topics[2].Bytes())
 		v := common.BytesToAddress(ethLog.Topics[3].Bytes())
 		wBytes := shared.GetDataBytesAtIndex(-3, ethLog.Data)

pkg/transformers/vat_init/converter.go

@@ -17,10 +17,10 @@
 package vat_init
 
 import (
-	"bytes"
 	"encoding/json"
 	"errors"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
 type VatInitConverter struct{}
@@ -32,7 +32,7 @@ func (VatInitConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
 		if err != nil {
 			return nil, err
 		}
-		ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
+		ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes())
 		raw, err := json.Marshal(ethLog)
 		if err != nil {
 			return nil, err

pkg/transformers/vat_slip/converter.go

@@ -17,11 +17,11 @@
 package vat_slip
 
 import (
-	"bytes"
 	"encoding/json"
 	"errors"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
 type VatSlipConverter struct{}
@@ -33,7 +33,7 @@ func (VatSlipConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
 		if err != nil {
 			return nil, err
 		}
-		ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
+		ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes())
 		guy := common.BytesToAddress(ethLog.Topics[2].Bytes())
 		rad := ethLog.Topics[3].Big()
 

pkg/transformers/vat_toll/converter.go

@@ -1,9 +1,9 @@
 package vat_toll
 
 import (
-	"bytes"
 	"encoding/json"
 	"errors"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -18,7 +18,7 @@ func (VatTollConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
 		if err != nil {
 			return nil, err
 		}
-		ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
+		ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes())
 		urn := common.BytesToAddress(ethLog.Topics[2].Bytes()[:common.AddressLength])
 		take := ethLog.Topics[3].Big()
 

pkg/transformers/vat_tune/converter.go

@@ -1,7 +1,6 @@
 package vat_tune
 
 import (
-	"bytes"
 	"encoding/json"
 	"errors"
 	"math/big"
@@ -22,7 +21,7 @@ func (VatTuneConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
 		if err != nil {
 			return nil, err
 		}
-		ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
+		ilk := shared.GetHexWithoutPrefix(ethLog.Topics[1].Bytes())
 		urn := common.BytesToAddress(ethLog.Topics[2].Bytes())
 		v := common.BytesToAddress(ethLog.Topics[3].Bytes())
 		wBytes := shared.GetDataBytesAtIndex(-3, ethLog.Data)