Merge branch 'master' into add_postgraphile_subscription

Gopkg.lock

@@ -225,6 +225,12 @@
   revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8"
   version = "v1.1.0"
 
+[[projects]]
+  name = "github.com/philhofer/fwd"
+  packages = ["."]
+  revision = "bb6d471dc95d4fe11e432687f8b70ff496cf3136"
+  version = "v1.0.0"
+
 [[projects]]
   name = "github.com/rjeczalik/notify"
   packages = ["."]
@@ -295,6 +301,12 @@
   ]
   revision = "adf24ef3f94bd13ec4163060b21a5678f22b429b"
 
+[[projects]]
+  name = "github.com/tinylib/msgp"
+  packages = ["msgp"]
+  revision = "b2b6a672cf1e5b90748f79b8b81fc8c5cf0571a1"
+  version = "1.0.2"
+
 [[projects]]
   branch = "master"
   name = "golang.org/x/crypto"
@@ -326,7 +338,10 @@
 [[projects]]
   branch = "master"
   name = "golang.org/x/sys"
-  packages = ["unix"]
+  packages = [
+    "unix",
+    "windows"
+  ]
   revision = "a0f4589a76f1f83070cb9e5613809e1d07b97c13"
 
 [[projects]]
@@ -366,6 +381,17 @@
   ]
   revision = "8cc4e8a6f4841aa92a8683fca47bc5d64b58875b"
 
+[[projects]]
+  name = "gopkg.in/DataDog/dd-trace-go.v1"
+  packages = [
+    "ddtrace",
+    "ddtrace/ext",
+    "ddtrace/internal",
+    "ddtrace/tracer"
+  ]
+  revision = "8efc9a798f2db99a9e00c7e57f45fc13611214e0"
+  version = "v1.2.3"
+
 [[projects]]
   name = "gopkg.in/fatih/set.v0"
   packages = ["."]
@@ -393,6 +419,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "2122d1d04fbc67daf75d751e144926e73f0fd838d51711cccf9604eabb46b95b"
+  inputs-digest = "7a913c984013e026536456baa75bd95e261bbb0d294b7de77785819ac182b465"
   solver-name = "gps-cdcl"
   solver-version = 1

README.md

@@ -17,6 +17,7 @@ Vulcanize DB is a set of tools that make it easier for developers to write appli
 
 ## Installation
 `go get github.com/vulcanize/vulcanizedb`
+`go get gopkg.in/DataDog/dd-trace-go.v1/ddtrace`
 
 ## Setting up the Database
 1. Install Postgres
@@ -27,6 +28,9 @@ Vulcanize DB is a set of tools that make it easier for developers to write appli
 
     * See below for configuring additional environments
 
+## Create a migration file (up and down)
+1. ./script/create_migrate create_bite_table
+
 ## Configuration
 - To use a local Ethereum node, copy `environments/public.toml.example` to
   `environments/public.toml` and update the `ipcPath` and `levelDbPath`.

cmd/root.go

@@ -70,6 +70,7 @@ func init() {
 	rootCmd.PersistentFlags().String("database-password", "", "database password")
 	rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
 	rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
+	rootCmd.PersistentFlags().String("datadog-name", "vulcanize-test", "datadog service name")
 
 	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
 	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
@@ -78,6 +79,7 @@ func init() {
 	viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
 	viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath"))
 	viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
+	viper.BindPFlag("datadog.name", rootCmd.PersistentFlags().Lookup("datadog-name"))
 }
 
 func initConfig() {

db/migrations/1532468319_create_flip_kick_table.up.sql

@@ -2,15 +2,11 @@ CREATE TABLE maker.flip_kick (
   db_id            SERIAL PRIMARY KEY,
   header_id        INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
   id               NUMERIC NOT NULL UNIQUE,
-  mom              VARCHAR,
-  vat              VARCHAR,
-  ilk              VARCHAR,
   lot              NUMERIC,
   bid              NUMERIC,
-  guy              VARCHAR,
   gal              VARCHAR,
 	"end"            TIMESTAMP WITH TIME ZONE,
-	era              TIMESTAMP WITH TIME ZONE,
+  urn              VARCHAR,
-  lad              VARCHAR,
+  tab              NUMERIC,
-  tab              NUMERIC
+  raw_log          JSONB
 );

db/migrations/1533844125_create_frob_table.up.sql

@@ -1,13 +1,14 @@
 CREATE TABLE maker.frob (
   id        SERIAL PRIMARY KEY,
   header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
-  tx_idx    INTEGER,
   ilk       bytea,
-  lad       bytea,
+  urn       bytea,
   dink      NUMERIC,
   dart      NUMERIC,
   ink       NUMERIC,
   art       NUMERIC,
   iart      NUMERIC,
+  tx_idx    INTEGER NOT NUll,
+  raw_log   JSONB,
   UNIQUE (header_id, tx_idx)
 );

db/migrations/1534193915_add_raw_log_column_to_flip_kick.down.sql

@@ -1,2 +0,0 @@
-ALTER TABLE maker.flip_kick
-  DROP COLUMN raw_log;

db/migrations/1534193915_add_raw_log_column_to_flip_kick.up.sql

@@ -1,2 +0,0 @@
-ALTER TABLE maker.flip_kick
-  ADD COLUMN raw_log json;

db/migrations/1534295712_create_tend_table.up.sql

@@ -1,12 +1,11 @@
 CREATE TABLE maker.tend (
   db_id            SERIAL PRIMARY KEY,
   header_id        INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
-  id               NUMERIC NOT NULL UNIQUE,
+  bid_id               NUMERIC NOT NULL UNIQUE,
   lot              NUMERIC,
   bid              NUMERIC,
-  guy              BYTEA,
+  guy              VARCHAR,
   tic              NUMERIC,
-	era              TIMESTAMP WITH TIME ZONE,
 	tx_idx           INTEGER NOT NUll,
   raw_log          JSONB
 );

db/migrations/1534295713_create_bite_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE maker.bite;

db/migrations/1534295713_create_bite_table.up.sql

@@ -0,0 +1,14 @@
+CREATE TABLE maker.bite (
+  id        SERIAL PRIMARY KEY,
+  header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
+  ilk       bytea,
+  lad       bytea,
+  ink       VARCHAR,
+  art       VARCHAR,
+  iArt      VARCHAR,
+  tab       NUMERIC,
+  flip      VARCHAR,
+  tx_idx    INTEGER NOT NUll,
+  raw_log   JSONB,
+  UNIQUE (header_id, tx_idx)
+)

db/migrations/1534455274_remove_mom_field_from_flip_kick.down.sql

@@ -1,2 +0,0 @@
-ALTER TABLE maker.flip_kick
-  ADD COLUMN mom VARCHAR;

db/migrations/1534455274_remove_mom_field_from_flip_kick.up.sql

@@ -1,2 +0,0 @@
-ALTER TABLE maker.flip_kick
-  DROP COLUMN mom;

db/migrations/1534799167_create_dent_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE maker.dent;

db/migrations/1534799167_create_dent_table.up.sql

@@ -0,0 +1,11 @@
+CREATE TABLE maker.dent (
+  db_id            SERIAL PRIMARY KEY,
+  header_id        INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
+  bid_id           NUMERIC NOT NULL UNIQUE,
+  lot              NUMERIC,
+  bid              NUMERIC,
+  guy              BYTEA,
+  tic              NUMERIC,
+	tx_idx           INTEGER NOT NUll,
+  raw_log          JSONB
+);

db/migrations/1535667935_create_pit_file_table.down.sql

@@ -0,0 +1,3 @@
+DROP TABLE maker.pit_file_ilk;
+DROP TABLE maker.pit_file_stability_fee;
+DROP TABLE maker.pit_file_debt_ceiling;

db/migrations/1535667935_create_pit_file_table.up.sql

@@ -0,0 +1,30 @@
+CREATE TABLE maker.pit_file_ilk (
+  id            SERIAL PRIMARY KEY,
+  header_id     INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
+  ilk           TEXT,
+  what          TEXT,
+  data          NUMERIC,
+	tx_idx        INTEGER NOT NUll,
+  raw_log       JSONB,
+  UNIQUE (header_id, tx_idx)
+);
+
+CREATE TABLE maker.pit_file_stability_fee (
+  id SERIAL PRIMARY KEY,
+  header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
+  what      TEXT,
+  data      TEXT,
+  tx_idx    INTEGER NOT NULL,
+  raw_log   JSONB,
+  UNIQUE (header_id, tx_idx)
+);
+
+CREATE TABLE maker.pit_file_debt_ceiling (
+  id SERIAL PRIMARY KEY,
+  header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
+  what      TEXT,
+  data      NUMERIC,
+  tx_idx    INTEGER NOT NULL,
+  raw_log   JSONB,
+  UNIQUE (header_id, tx_idx)
+);

db/migrations/1536267596_create_vat_init_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE maker.vat_init;

db/migrations/1536267596_create_vat_init_table.up.sql

@@ -0,0 +1,8 @@
+CREATE TABLE maker.vat_init (
+  id            SERIAL PRIMARY KEY,
+  header_id     INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
+  ilk           TEXT,
+	tx_idx        INTEGER NOT NUll,
+  raw_log       JSONB,
+  UNIQUE (header_id, tx_idx)
+);

db/schema.sql

@@ -56,6 +56,82 @@ SET default_tablespace = '';
 
 SET default_with_oids = false;
 
+--
+-- Name: bite; Type: TABLE; Schema: maker; Owner: -
+--
+
+CREATE TABLE maker.bite (
+    id integer NOT NULL,
+    header_id integer NOT NULL,
+    ilk bytea,
+    lad bytea,
+    ink character varying,
+    art character varying,
+    iart character varying,
+    tab numeric,
+    flip character varying,
+    tx_idx integer NOT NULL,
+    raw_log jsonb
+);
+
+
+--
+-- Name: bite_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
+--
+
+CREATE SEQUENCE maker.bite_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: bite_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
+--
+
+ALTER SEQUENCE maker.bite_id_seq OWNED BY maker.bite.id;
+
+
+--
+-- Name: dent; Type: TABLE; Schema: maker; Owner: -
+--
+
+CREATE TABLE maker.dent (
+    db_id integer NOT NULL,
+    header_id integer NOT NULL,
+    bid_id numeric NOT NULL,
+    lot numeric,
+    bid numeric,
+    guy bytea,
+    tic numeric,
+    tx_idx integer NOT NULL,
+    raw_log jsonb
+);
+
+
+--
+-- Name: dent_db_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
+--
+
+CREATE SEQUENCE maker.dent_db_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: dent_db_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
+--
+
+ALTER SEQUENCE maker.dent_db_id_seq OWNED BY maker.dent.db_id;
+
+
 --
 -- Name: flip_kick; Type: TABLE; Schema: maker; Owner: -
 --
@@ -64,17 +140,13 @@ CREATE TABLE maker.flip_kick (
     db_id integer NOT NULL,
     header_id integer NOT NULL,
     id numeric NOT NULL,
-    vat character varying,
-    ilk character varying,
     lot numeric,
     bid numeric,
-    guy character varying,
     gal character varying,
     "end" timestamp with time zone,
-    era timestamp with time zone,
+    urn character varying,
-    lad character varying,
     tab numeric,
-    raw_log json
+    raw_log jsonb
 );
 
 
@@ -105,14 +177,15 @@ ALTER SEQUENCE maker.flip_kick_db_id_seq OWNED BY maker.flip_kick.db_id;
 CREATE TABLE maker.frob (
     id integer NOT NULL,
     header_id integer NOT NULL,
-    tx_idx integer,
     ilk bytea,
-    lad bytea,
+    urn bytea,
     dink numeric,
     dart numeric,
     ink numeric,
     art numeric,
-    iart numeric
+    iart numeric,
+    tx_idx integer NOT NULL,
+    raw_log jsonb
 );
 
 
@@ -136,6 +209,109 @@ CREATE SEQUENCE maker.frob_id_seq
 ALTER SEQUENCE maker.frob_id_seq OWNED BY maker.frob.id;
 
 
+--
+-- Name: pit_file_debt_ceiling; Type: TABLE; Schema: maker; Owner: -
+--
+
+CREATE TABLE maker.pit_file_debt_ceiling (
+    id integer NOT NULL,
+    header_id integer NOT NULL,
+    what text,
+    data numeric,
+    tx_idx integer NOT NULL,
+    raw_log jsonb
+);
+
+
+--
+-- Name: pit_file_debt_ceiling_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
+--
+
+CREATE SEQUENCE maker.pit_file_debt_ceiling_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: pit_file_debt_ceiling_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
+--
+
+ALTER SEQUENCE maker.pit_file_debt_ceiling_id_seq OWNED BY maker.pit_file_debt_ceiling.id;
+
+
+--
+-- Name: pit_file_ilk; Type: TABLE; Schema: maker; Owner: -
+--
+
+CREATE TABLE maker.pit_file_ilk (
+    id integer NOT NULL,
+    header_id integer NOT NULL,
+    ilk text,
+    what text,
+    data numeric,
+    tx_idx integer NOT NULL,
+    raw_log jsonb
+);
+
+
+--
+-- Name: pit_file_ilk_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
+--
+
+CREATE SEQUENCE maker.pit_file_ilk_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: pit_file_ilk_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
+--
+
+ALTER SEQUENCE maker.pit_file_ilk_id_seq OWNED BY maker.pit_file_ilk.id;
+
+
+--
+-- Name: pit_file_stability_fee; Type: TABLE; Schema: maker; Owner: -
+--
+
+CREATE TABLE maker.pit_file_stability_fee (
+    id integer NOT NULL,
+    header_id integer NOT NULL,
+    what text,
+    data text,
+    tx_idx integer NOT NULL,
+    raw_log jsonb
+);
+
+
+--
+-- Name: pit_file_stability_fee_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
+--
+
+CREATE SEQUENCE maker.pit_file_stability_fee_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: pit_file_stability_fee_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
+--
+
+ALTER SEQUENCE maker.pit_file_stability_fee_id_seq OWNED BY maker.pit_file_stability_fee.id;
+
+
 --
 -- Name: price_feeds; Type: TABLE; Schema: maker; Owner: -
 --
@@ -177,12 +353,11 @@ ALTER SEQUENCE maker.price_feeds_id_seq OWNED BY maker.price_feeds.id;
 CREATE TABLE maker.tend (
     db_id integer NOT NULL,
     header_id integer NOT NULL,
-    id numeric NOT NULL,
+    bid_id numeric NOT NULL,
     lot numeric,
     bid numeric,
-    guy bytea,
+    guy character varying,
     tic numeric,
-    era timestamp with time zone,
     tx_idx integer NOT NULL,
     raw_log jsonb
 );
@@ -208,6 +383,39 @@ CREATE SEQUENCE maker.tend_db_id_seq
 ALTER SEQUENCE maker.tend_db_id_seq OWNED BY maker.tend.db_id;
 
 
+--
+-- Name: vat_init; Type: TABLE; Schema: maker; Owner: -
+--
+
+CREATE TABLE maker.vat_init (
+    id integer NOT NULL,
+    header_id integer NOT NULL,
+    ilk text,
+    tx_idx integer NOT NULL,
+    raw_log jsonb
+);
+
+
+--
+-- Name: vat_init_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
+--
+
+CREATE SEQUENCE maker.vat_init_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: vat_init_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
+--
+
+ALTER SEQUENCE maker.vat_init_id_seq OWNED BY maker.vat_init.id;
+
+
 --
 -- Name: logs; Type: TABLE; Schema: public; Owner: -
 --
@@ -580,6 +788,20 @@ CREATE VIEW public.watched_event_logs AS
   WHERE ((((log_filters.topic0)::text = (logs.topic0)::text) OR (log_filters.topic0 IS NULL)) AND (((log_filters.topic1)::text = (logs.topic1)::text) OR (log_filters.topic1 IS NULL)) AND (((log_filters.topic2)::text = (logs.topic2)::text) OR (log_filters.topic2 IS NULL)) AND (((log_filters.topic3)::text = (logs.topic3)::text) OR (log_filters.topic3 IS NULL)));
 
 
+--
+-- Name: bite id; Type: DEFAULT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.bite ALTER COLUMN id SET DEFAULT nextval('maker.bite_id_seq'::regclass);
+
+
+--
+-- Name: dent db_id; Type: DEFAULT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.dent ALTER COLUMN db_id SET DEFAULT nextval('maker.dent_db_id_seq'::regclass);
+
+
 --
 -- Name: flip_kick db_id; Type: DEFAULT; Schema: maker; Owner: -
 --
@@ -594,6 +816,27 @@ ALTER TABLE ONLY maker.flip_kick ALTER COLUMN db_id SET DEFAULT nextval('maker.f
 ALTER TABLE ONLY maker.frob ALTER COLUMN id SET DEFAULT nextval('maker.frob_id_seq'::regclass);
 
 
+--
+-- Name: pit_file_debt_ceiling id; Type: DEFAULT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_file_debt_ceiling ALTER COLUMN id SET DEFAULT nextval('maker.pit_file_debt_ceiling_id_seq'::regclass);
+
+
+--
+-- Name: pit_file_ilk id; Type: DEFAULT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_file_ilk ALTER COLUMN id SET DEFAULT nextval('maker.pit_file_ilk_id_seq'::regclass);
+
+
+--
+-- Name: pit_file_stability_fee id; Type: DEFAULT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_file_stability_fee ALTER COLUMN id SET DEFAULT nextval('maker.pit_file_stability_fee_id_seq'::regclass);
+
+
 --
 -- Name: price_feeds id; Type: DEFAULT; Schema: maker; Owner: -
 --
@@ -608,6 +851,13 @@ ALTER TABLE ONLY maker.price_feeds ALTER COLUMN id SET DEFAULT nextval('maker.pr
 ALTER TABLE ONLY maker.tend ALTER COLUMN db_id SET DEFAULT nextval('maker.tend_db_id_seq'::regclass);
 
 
+--
+-- Name: vat_init id; Type: DEFAULT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.vat_init ALTER COLUMN id SET DEFAULT nextval('maker.vat_init_id_seq'::regclass);
+
+
 --
 -- Name: blocks id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -671,6 +921,38 @@ ALTER TABLE ONLY public.transactions ALTER COLUMN id SET DEFAULT nextval('public
 ALTER TABLE ONLY public.watched_contracts ALTER COLUMN contract_id SET DEFAULT nextval('public.watched_contracts_contract_id_seq'::regclass);
 
 
+--
+-- Name: bite bite_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.bite
+    ADD CONSTRAINT bite_header_id_tx_idx_key UNIQUE (header_id, tx_idx);
+
+
+--
+-- Name: bite bite_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.bite
+    ADD CONSTRAINT bite_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: dent dent_bid_id_key; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.dent
+    ADD CONSTRAINT dent_bid_id_key UNIQUE (bid_id);
+
+
+--
+-- Name: dent dent_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.dent
+    ADD CONSTRAINT dent_pkey PRIMARY KEY (db_id);
+
+
 --
 -- Name: flip_kick flip_kick_id_key; Type: CONSTRAINT; Schema: maker; Owner: -
 --
@@ -703,6 +985,54 @@ ALTER TABLE ONLY maker.frob
     ADD CONSTRAINT frob_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: pit_file_debt_ceiling pit_file_debt_ceiling_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_file_debt_ceiling
+    ADD CONSTRAINT pit_file_debt_ceiling_header_id_tx_idx_key UNIQUE (header_id, tx_idx);
+
+
+--
+-- Name: pit_file_debt_ceiling pit_file_debt_ceiling_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_file_debt_ceiling
+    ADD CONSTRAINT pit_file_debt_ceiling_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: pit_file_ilk pit_file_ilk_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_file_ilk
+    ADD CONSTRAINT pit_file_ilk_header_id_tx_idx_key UNIQUE (header_id, tx_idx);
+
+
+--
+-- Name: pit_file_ilk pit_file_ilk_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_file_ilk
+    ADD CONSTRAINT pit_file_ilk_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: pit_file_stability_fee pit_file_stability_fee_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_file_stability_fee
+    ADD CONSTRAINT pit_file_stability_fee_header_id_tx_idx_key UNIQUE (header_id, tx_idx);
+
+
+--
+-- Name: pit_file_stability_fee pit_file_stability_fee_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_file_stability_fee
+    ADD CONSTRAINT pit_file_stability_fee_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: price_feeds price_feeds_header_id_medianizer_address_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: -
 --
@@ -720,11 +1050,11 @@ ALTER TABLE ONLY maker.price_feeds
 
 
 --
--- Name: tend tend_id_key; Type: CONSTRAINT; Schema: maker; Owner: -
+-- Name: tend tend_bid_id_key; Type: CONSTRAINT; Schema: maker; Owner: -
 --
 
 ALTER TABLE ONLY maker.tend
-    ADD CONSTRAINT tend_id_key UNIQUE (id);
+    ADD CONSTRAINT tend_bid_id_key UNIQUE (bid_id);
 
 
 --
@@ -735,6 +1065,22 @@ ALTER TABLE ONLY maker.tend
     ADD CONSTRAINT tend_pkey PRIMARY KEY (db_id);
 
 
+--
+-- Name: vat_init vat_init_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.vat_init
+    ADD CONSTRAINT vat_init_header_id_tx_idx_key UNIQUE (header_id, tx_idx);
+
+
+--
+-- Name: vat_init vat_init_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.vat_init
+    ADD CONSTRAINT vat_init_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -873,6 +1219,22 @@ CREATE INDEX tx_to_index ON public.transactions USING btree (tx_to);
 CREATE TRIGGER notify_pricefeeds AFTER INSERT ON maker.price_feeds FOR EACH ROW EXECUTE PROCEDURE public.notify_pricefeed();
 
 
+--
+-- Name: bite bite_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.bite
+    ADD CONSTRAINT bite_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
+
+
+--
+-- Name: dent dent_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.dent
+    ADD CONSTRAINT dent_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
+
+
 --
 -- Name: flip_kick flip_kick_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: -
 --
@@ -897,6 +1259,30 @@ ALTER TABLE ONLY maker.price_feeds
     ADD CONSTRAINT headers_fk FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
 
 
+--
+-- Name: pit_file_debt_ceiling pit_file_debt_ceiling_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_file_debt_ceiling
+    ADD CONSTRAINT pit_file_debt_ceiling_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
+
+
+--
+-- Name: pit_file_ilk pit_file_ilk_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_file_ilk
+    ADD CONSTRAINT pit_file_ilk_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
+
+
+--
+-- Name: pit_file_stability_fee pit_file_stability_fee_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.pit_file_stability_fee
+    ADD CONSTRAINT pit_file_stability_fee_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
+
+
 --
 -- Name: tend tend_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: -
 --
@@ -905,6 +1291,14 @@ ALTER TABLE ONLY maker.tend
     ADD CONSTRAINT tend_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
 
 
+--
+-- Name: vat_init vat_init_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.vat_init
+    ADD CONSTRAINT vat_init_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
+
+
 --
 -- Name: transactions blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
 --

environments/staging.toml

@@ -7,3 +7,6 @@ port = 5432
 
 [client]
 ipcPath = "/mnt/geth.ipc"
+
+[datadog]
+name = "maker_vdb_staging"

main.go

@@ -2,8 +2,15 @@ package main
 
 import (
 	"github.com/vulcanize/vulcanizedb/cmd"
+
+	"github.com/spf13/viper"
+	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
 )
 
 func main() {
+	tracer.Start(tracer.WithServiceName(viper.GetString("datadog.name")))
+
 	cmd.Execute()
+
+	defer tracer.Stop()
 }

pkg/transformers/bite/bite_suite_test.go

@@ -0,0 +1,33 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bite_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"io/ioutil"
+	"log"
+)
+
+func TestBite(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Bite Suite")
+}
+
+var _ = BeforeSuite(func() {
+	log.SetOutput(ioutil.Discard)
+})

pkg/transformers/bite/config.go

@@ -0,0 +1,29 @@
+/*
+ *  Copyright 2018 Vulcanize
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package bite
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+)
+
+var BiteConfig = shared.TransformerConfig{
+	ContractAddress:     shared.CatContractAddress,
+	ContractAbi:         shared.CatABI,
+	Topics:              []string{shared.BiteSignature},
+	StartingBlockNumber: 0,
+	EndingBlockNumber:   100,
+}

pkg/transformers/bite/converter.go

@@ -0,0 +1,84 @@
+/*
+ *  Copyright 2018 Vulcanize
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package bite
+
+import (
+	"encoding/json"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+)
+
+type Converter interface {
+	ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (BiteEntity, error)
+	ToModel(flipKick BiteEntity) (BiteModel, error)
+}
+
+type BiteConverter struct{}
+
+func (BiteConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (BiteEntity, error) {
+	entity := BiteEntity{}
+	address := common.HexToAddress(contractAddress)
+	abi, err := geth.ParseAbi(contractAbi)
+	if err != nil {
+		return entity, err
+	}
+
+	contract := bind.NewBoundContract(address, abi, nil, nil, nil)
+
+	err = contract.UnpackLog(&entity, "Bite", ethLog)
+	if err != nil {
+		return entity, err
+	}
+
+	entity.Raw = ethLog
+	entity.TransactionIndex = ethLog.TxIndex
+
+	return entity, nil
+}
+func (converter BiteConverter) ToModel(entity BiteEntity) (BiteModel, error) {
+
+	id := entity.Id.String()
+	ilk := entity.Ilk[:]
+	lad := entity.Lad[:]
+	ink := entity.Ink.String()
+	art := entity.Art.String()
+	iArt := entity.IArt.String()
+	tab := entity.Tab.String()
+	flip := entity.Flip.String()
+	txIdx := entity.TransactionIndex
+	rawLogJson, err := json.Marshal(entity.Raw)
+	rawLogString := string(rawLogJson)
+	if err != nil {
+		return BiteModel{}, err
+	}
+
+	return BiteModel{
+		Id:               shared.ConvertNilToEmptyString(id),
+		Ilk:              ilk,
+		Lad:              lad,
+		Ink:              shared.ConvertNilToEmptyString(ink),
+		Art:              shared.ConvertNilToEmptyString(art),
+		IArt:             shared.ConvertNilToEmptyString(iArt),
+		Tab:              shared.ConvertNilToEmptyString(tab),
+		Flip:             shared.ConvertNilToEmptyString(flip),
+		TransactionIndex: txIdx,
+		Raw:              rawLogString,
+	}, nil
+}

pkg/transformers/bite/converter_test.go

@@ -0,0 +1,94 @@
+/*
+ *  Copyright 2018 Vulcanize
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package bite_test
+
+import (
+	"encoding/json"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+)
+
+var _ = Describe("Bite Converter", func() {
+	var converter = bite.BiteConverter{}
+
+	Describe("ToEntity", func() {
+		It("converts an eth log to a bite entity", func() {
+			entity, err := converter.ToEntity(shared.CatContractAddress, shared.CatABI, test_data.EthBiteLog)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(entity.Ilk).To(Equal(test_data.BiteEntity.Ilk))
+			Expect(entity.Lad).To(Equal(test_data.BiteEntity.Lad))
+			Expect(entity.Ink).To(Equal(test_data.BiteEntity.Ink))
+			Expect(entity.Art).To(Equal(test_data.BiteEntity.Art))
+			Expect(entity.Tab).To(Equal(test_data.BiteEntity.Tab))
+			Expect(entity.Flip).To(Equal(test_data.BiteEntity.Flip))
+			Expect(entity.IArt).To(Equal(test_data.BiteEntity.IArt))
+			Expect(entity.TransactionIndex).To(Equal(test_data.BiteEntity.TransactionIndex))
+			Expect(entity.Raw).To(Equal(test_data.BiteEntity.Raw))
+		})
+
+		It("returns an error if converting log to entity fails", func() {
+			_, err := converter.ToEntity(shared.CatContractAddress, "error abi", test_data.EthBiteLog)
+
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Describe("ToModel", func() {
+		var emptyEntity = bite.BiteEntity{}
+
+		BeforeEach(func() {
+			emptyEntity.Id = big.NewInt(1)
+		})
+
+		It("converts an Entity to a Model", func() {
+			model, err := converter.ToModel(test_data.BiteEntity)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(model).To(Equal(test_data.BiteModel))
+			Expect(model.TransactionIndex).To(Equal(test_data.BiteModel.TransactionIndex))
+		})
+
+		It("handles nil values", func() {
+			emptyLog, err := json.Marshal(types.Log{})
+			Expect(err).NotTo(HaveOccurred())
+			expectedModel := bite.BiteModel{
+				Id:               "1",
+				Ilk:              []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+				Lad:              []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+				Ink:              "",
+				Art:              "",
+				IArt:             "",
+				Tab:              "",
+				Flip:             "",
+				TransactionIndex: 0,
+				Raw:              string(emptyLog),
+			}
+			model, err := converter.ToModel(emptyEntity)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(model).To(Equal(expectedModel))
+		})
+	})
+})

pkg/transformers/bite/entity.go

@@ -12,22 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package tend
+package bite
 
 import (
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"math/big"
 )
 
-type TendEntity struct {
+type BiteEntity struct {
 	Id               *big.Int
-	Lot              *big.Int
+	Ilk              [32]byte
-	Bid              *big.Int
+	Lad              [32]byte
-	Guy              common.Address
+	Ink              *big.Int
-	Tic              *big.Int
+	Art              *big.Int
-	Era              *big.Int
+	Tab              *big.Int
+	Flip             *big.Int
+	IArt             *big.Int
 	TransactionIndex uint
 	Raw              types.Log
 }

pkg/transformers/bite/integration_test.go

@@ -0,0 +1,83 @@
+/*
+ *  Copyright 2018 Vulcanize
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package bite_test
+
+import (
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethclient"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
+	rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Integration tests", func() {
+	XIt("Fetches bite event logs from a local test chain", func() {
+		ipcPath := test_config.TestClient.IPCPath
+
+		rawRpcClient, err := rpc.Dial(ipcPath)
+		Expect(err).NotTo(HaveOccurred())
+
+		rpcClient := client.NewRpcClient(rawRpcClient, ipcPath)
+		ethClient := ethclient.NewClient(rawRpcClient)
+		blockChainClient := client.NewEthClient(ethClient)
+		realNode := node.MakeNode(rpcClient)
+		transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
+		realBlockChain := geth.NewBlockChain(blockChainClient, realNode, transactionConverter)
+		realFetcher := shared.NewFetcher(realBlockChain)
+		topic0 := common.HexToHash(shared.BiteSignature)
+		topics := [][]common.Hash{{topic0}}
+
+		result, err := realFetcher.FetchLogs(shared.CatContractAddress, topics, int64(26))
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(len(result) > 0).To(BeTrue())
+		Expect(result[0].Address).To(Equal(common.HexToAddress(shared.CatContractAddress)))
+		Expect(result[0].TxHash).To(Equal(test_data.EthBiteLog.TxHash))
+		Expect(result[0].BlockNumber).To(Equal(test_data.EthBiteLog.BlockNumber))
+		Expect(result[0].Topics).To(Equal(test_data.EthBiteLog.Topics))
+		Expect(result[0].Index).To(Equal(test_data.EthBiteLog.Index))
+	})
+
+	It("unpacks an event log", func() {
+		address := common.HexToAddress(shared.CatContractAddress)
+		abi, err := geth.ParseAbi(shared.CatABI)
+		Expect(err).NotTo(HaveOccurred())
+
+		contract := bind.NewBoundContract(address, abi, nil, nil, nil)
+		entity := &bite.BiteEntity{}
+
+		var eventLog = test_data.EthBiteLog
+
+		err = contract.UnpackLog(entity, "Bite", eventLog)
+		Expect(err).NotTo(HaveOccurred())
+
+		expectedEntity := test_data.BiteEntity
+		Expect(entity.Art).To(Equal(expectedEntity.Art))
+		Expect(entity.Ilk).To(Equal(expectedEntity.Ilk))
+		Expect(entity.Ink).To(Equal(expectedEntity.Ink))
+	})
+})

pkg/transformers/bite/model.go

@@ -0,0 +1,28 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bite
+
+type BiteModel struct {
+	Id               string
+	Ilk              []byte
+	Lad              []byte
+	Ink              string
+	Art              string
+	IArt             string
+	Tab              string
+	Flip             string
+	TransactionIndex uint   `db:"tx_idx"`
+	Raw              string `db:"raw_log"`
+}

pkg/transformers/bite/repository.go

@@ -0,0 +1,63 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bite
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type Repository interface {
+	Create(headerID int64, model BiteModel) error
+	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
+}
+
+type BiteRepository struct {
+	db *postgres.DB
+}
+
+func NewBiteRepository(db *postgres.DB) Repository {
+	return BiteRepository{db: db}
+}
+
+func (repository BiteRepository) Create(headerID int64, model BiteModel) error {
+	_, err := repository.db.Exec(
+		`INSERT into maker.bite (header_id, id, ilk, lad, ink, art, iart, tab, flip, tx_idx, raw_log)
+        VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`,
+		headerID, model.Id, model.Ilk, model.Lad, model.Ink, model.Art, model.IArt, model.Tab, model.Flip, model.TransactionIndex, model.Raw,
+	)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+func (repository BiteRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) {
+	var result []core.Header
+	err := repository.db.Select(
+		&result,
+		`SELECT headers.id, headers.block_number FROM headers
+               LEFT JOIN maker.bite on headers.id = header_id
+               WHERE header_id ISNULL
+               AND headers.block_number >= $1
+               AND headers.block_number <= $2
+               AND headers.eth_node_fingerprint = $3`,
+		startingBlockNumber,
+		endingBlockNumber,
+		repository.db.Node.ID,
+	)
+	return result, err
+}

pkg/transformers/bite/repository_test.go

@@ -0,0 +1,135 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bite_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"database/sql"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Bite repository", func() {
+	Describe("Create", func() {
+		It("persists a bite record", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			biteRepository := bite.NewBiteRepository(db)
+
+			err = biteRepository.Create(headerID, test_data.BiteModel)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbBite bite.BiteModel
+			err = db.Get(&dbBite, `SELECT id, ilk, lad, ink, art, tab, flip, tx_idx, raw_log FROM maker.bite WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(dbBite.Id).To(Equal(test_data.BiteModel.Id))
+			Expect(dbBite.Ilk).To(Equal(test_data.BiteModel.Ilk))
+			Expect(dbBite.Lad).To(Equal(test_data.BiteModel.Lad))
+			Expect(dbBite.Art).To(Equal(test_data.BiteModel.Art))
+			Expect(dbBite.Tab).To(Equal(test_data.BiteModel.Tab))
+			Expect(dbBite.Flip).To(Equal(test_data.BiteModel.Flip))
+			Expect(dbBite.TransactionIndex).To(Equal(test_data.BiteModel.TransactionIndex))
+			Expect(dbBite.Raw).To(MatchJSON(test_data.BiteModel.Raw))
+		})
+
+		It("does not duplicate bite events", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			biteRepository := bite.NewBiteRepository(db)
+			err = biteRepository.Create(headerID, test_data.BiteModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			var anotherBiteModel = bite.BiteModel{
+				Id:               "11",
+				Ilk:              test_data.BiteModel.Ilk,
+				Lad:              test_data.BiteModel.Lad,
+				Ink:              test_data.BiteModel.Ink,
+				Art:              test_data.BiteModel.Art,
+				Tab:              test_data.BiteModel.Tab,
+				Flip:             test_data.BiteModel.Flip,
+				IArt:             test_data.BiteModel.IArt,
+				TransactionIndex: test_data.BiteModel.TransactionIndex,
+				Raw:              test_data.BiteModel.Raw,
+			}
+
+			err = biteRepository.Create(headerID, anotherBiteModel)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
+		})
+
+		It("removes bite if corresponding header is deleted", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			biteRepository := bite.NewBiteRepository(db)
+			err = biteRepository.Create(headerID, test_data.BiteModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbBite bite.BiteModel
+			err = db.Get(&dbBite, `SELECT id, ilk, lad, ink, art, tab, flip, tx_idx, raw_log FROM maker.bite WHERE header_id = $1`, headerID)
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(sql.ErrNoRows))
+		})
+	})
+
+	Describe("MissingHeaders", func() {
+		It("returns headers with no associated bite event", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			startingBlockNumber := int64(1)
+			biteBlockNumber := int64(2)
+			endingBlockNumber := int64(3)
+			blockNumbers := []int64{startingBlockNumber, biteBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			biteRepository := bite.NewBiteRepository(db)
+			err := biteRepository.Create(headerIDs[1], test_data.BiteModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := biteRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(2))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+		})
+	})
+})

pkg/transformers/bite/transformer.go

@@ -0,0 +1,88 @@
+/*
+ *  Copyright 2018 Vulcanize
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package bite
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+	"log"
+)
+
+type BiteTransformer struct {
+	Repository Repository
+	Fetcher    shared.LogFetcher
+	Converter  Converter
+	Config     shared.TransformerConfig
+}
+
+type BiteTransformerInitializer struct {
+	Config shared.TransformerConfig
+}
+
+func (i BiteTransformerInitializer) NewBiteTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
+	fetcher := shared.NewFetcher(blockChain)
+	repository := NewBiteRepository(db)
+	transformer := BiteTransformer{
+		Fetcher:    fetcher,
+		Repository: repository,
+		Converter:  BiteConverter{},
+		Config:     i.Config,
+	}
+
+	return transformer
+}
+
+func (b BiteTransformer) Execute() error {
+	config := b.Config
+	topics := [][]common.Hash{{common.HexToHash(shared.BiteSignature)}}
+
+	missingHeaders, err := b.Repository.MissingHeaders(config.StartingBlockNumber, config.EndingBlockNumber)
+	if err != nil {
+		log.Println("Error fetching missing headers:", err)
+		return err
+	}
+
+	for _, header := range missingHeaders {
+		ethLogs, err := b.Fetcher.FetchLogs(config.ContractAddress, topics, header.BlockNumber)
+		if err != nil {
+			log.Println("Error fetching matching logs:", err)
+			return err
+		}
+
+		for _, ethLog := range ethLogs {
+			entity, err := b.Converter.ToEntity(config.ContractAddress, config.ContractAbi, ethLog)
+			model, err := b.Converter.ToModel(entity)
+			if err != nil {
+				log.Println("Error converting logs:", err)
+				return err
+			}
+
+			err = b.Repository.Create(header.Id, model)
+			if err != nil {
+				log.Println("Error persisting bite record:", err)
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+func (b BiteTransformer) SetConfig(config shared.TransformerConfig) {
+	b.Config = config
+}

pkg/transformers/bite/transfromer_test.go

@@ -0,0 +1,144 @@
+/*
+ *  Copyright 2018 Vulcanize
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package bite_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
+	bite_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/bite"
+	"math/rand"
+)
+
+var _ = Describe("Bite Transformer", func() {
+	var repository bite_mocks.MockBiteRepository
+	var fetcher mocks.MockLogFetcher
+	var converter bite_mocks.MockBiteConverter
+	var transformer bite.BiteTransformer
+	var blockNumber1 = rand.Int63()
+	var blockNumber2 = rand.Int63()
+	var testConfig shared.TransformerConfig
+
+	BeforeEach(func() {
+		repository = bite_mocks.MockBiteRepository{}
+		fetcher = mocks.MockLogFetcher{}
+		converter = bite_mocks.MockBiteConverter{}
+
+		transformer = bite.BiteTransformer{
+			Repository: &repository,
+			Fetcher:    &fetcher,
+			Converter:  &converter,
+			Config:     bite.BiteConfig,
+		}
+
+		testConfig = shared.TransformerConfig{
+			ContractAddress:     "0x12345",
+			ContractAbi:         "test abi",
+			Topics:              []string{shared.BiteSignature},
+			StartingBlockNumber: blockNumber1,
+			EndingBlockNumber:   blockNumber2,
+		}
+		transformer.SetConfig(testConfig)
+	})
+
+	It("gets missing headers for blocks in the configured range", func() {
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.PassedStartingBlockNumber).To(Equal(bite.BiteConfig.StartingBlockNumber))
+		Expect(repository.PassedEndingBlockNumber).To(Equal(bite.BiteConfig.EndingBlockNumber))
+	})
+
+	It("returns an error if it fails to get missing headers", func() {
+		repository.SetMissingHeadersErr(fakes.FakeError)
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("fetches eth logs for each missing header", func() {
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: blockNumber1}, {BlockNumber: blockNumber2}})
+		expectedTopics := [][]common.Hash{{common.HexToHash(shared.BiteSignature)}}
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fetcher.FetchedBlocks).To(Equal([]int64{blockNumber1, blockNumber2}))
+		Expect(fetcher.FetchedTopics).To(Equal(expectedTopics))
+		Expect(fetcher.FetchedContractAddress).To(Equal(bite.BiteConfig.ContractAddress))
+	})
+
+	It("returns an error if fetching logs fails", func() {
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		fetcher.SetFetcherError(fakes.FakeError)
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("converts an eth log to an Entity", func() {
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthBiteLog})
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(converter.ConverterContract).To(Equal(bite.BiteConfig.ContractAddress))
+		Expect(converter.ConverterAbi).To(Equal(bite.BiteConfig.ContractAbi))
+		Expect(converter.LogsToConvert).To(Equal([]types.Log{test_data.EthBiteLog}))
+	})
+
+	It("returns an error if converter fails", func() {
+		headerId := int64(1)
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: blockNumber1, Id: headerId}})
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthBiteLog})
+		converter.SetConverterError(fakes.FakeError)
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("persists the bite record", func() {
+		headerId := int64(1)
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: blockNumber1, Id: headerId}})
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthBiteLog})
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.PassedHeaderID).To(Equal(headerId))
+		Expect(repository.PassedBiteModel).To(Equal(test_data.BiteModel))
+	})
+
+	It("returns error if persisting bite record fails", func() {
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: blockNumber1}})
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthBiteLog})
+		repository.SetCreateError(fakes.FakeError)
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+})

pkg/transformers/dent/config.go

@@ -0,0 +1,25 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dent
+
+import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+
+var DentConfig = shared.TransformerConfig{
+	ContractAddress:     shared.FlipperContractAddress,
+	ContractAbi:         shared.FlipperABI,
+	Topics:              []string{shared.DentFunctionSignature},
+	StartingBlockNumber: 0,
+	EndingBlockNumber:   100,
+}

pkg/transformers/dent/converter.go

@@ -0,0 +1,87 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dent
+
+import (
+	"encoding/json"
+	"errors"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+type Converter interface {
+	Convert(contractAddress string, contractAbi string, ethLog types.Log) (DentModel, error)
+}
+
+type DentConverter struct{}
+
+func NewDentConverter() DentConverter {
+	return DentConverter{}
+}
+
+func (c DentConverter) Convert(contractAddress, contractAbi string, ethLog types.Log) (DentModel, error) {
+	err := validateLog(ethLog)
+	if err != nil {
+		return DentModel{}, err
+	}
+
+	bidId := ethLog.Topics[2].Big()
+	lot := ethLog.Topics[3].Big().String()
+	bidValue := getBidValue(ethLog)
+	guy := common.HexToAddress(ethLog.Topics[1].Hex()).String()
+	tic := "0"
+	//TODO: it is likely that the tic value will need to be added to an emitted event,
+	//so this will need to be updated at that point
+
+	transactionIndex := ethLog.TxIndex
+
+	raw, err := json.Marshal(ethLog)
+	if err != nil {
+		return DentModel{}, err
+	}
+
+	return DentModel{
+		BidId:            bidId.String(),
+		Lot:              lot,
+		Bid:              bidValue,
+		Guy:              guy,
+		Tic:              tic,
+		TransactionIndex: transactionIndex,
+		Raw:              raw,
+	}, nil
+}
+
+func validateLog(ethLog types.Log) error {
+	if len(ethLog.Data) <= 0 {
+		return errors.New("dent log data is empty")
+	}
+
+	if len(ethLog.Topics) < 4 {
+		return errors.New("dent log does not contain expected topics")
+	}
+
+	return nil
+}
+
+func getBidValue(ethLog types.Log) string {
+	itemByteLength := 32
+	lastDataItemStartIndex := len(ethLog.Data) - itemByteLength
+	lastItem := ethLog.Data[lastDataItemStartIndex:]
+	lastValue := big.NewInt(0).SetBytes(lastItem)
+
+	return lastValue.String()
+}

pkg/transformers/dent/converter_test.go

@@ -0,0 +1,58 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dent_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/dent"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+)
+
+var _ = Describe("Dent Converter", func() {
+	var converter dent.DentConverter
+
+	BeforeEach(func() {
+		converter = dent.NewDentConverter()
+	})
+
+	It("converts an eth log to a db model", func() {
+		model, err := converter.Convert(shared.FlipperContractAddress, shared.FlipperABI, test_data.DentLog)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(model).To(Equal(test_data.DentModel))
+	})
+
+	It("returns an error if the expected amount of topics aren't in the log", func() {
+		invalidLog := test_data.DentLog
+		invalidLog.Topics = []common.Hash{}
+		model, err := converter.Convert(shared.FlipperContractAddress, shared.FlipperABI, invalidLog)
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError("dent log does not contain expected topics"))
+		Expect(model).To(Equal(dent.DentModel{}))
+	})
+
+	It("returns an error if the log data is empty", func() {
+		emptyDataLog := test_data.DentLog
+		emptyDataLog.Data = []byte{}
+		model, err := converter.Convert(shared.FlipperContractAddress, shared.FlipperABI, emptyDataLog)
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError("dent log data is empty"))
+		Expect(model).To(Equal(dent.DentModel{}))
+	})
+})

pkg/transformers/dent/dent_suite_test.go

@@ -0,0 +1,19 @@
+package dent_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"io/ioutil"
+	"log"
+)
+
+func TestDent(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Dent Suite")
+}
+
+var _ = BeforeSuite(func() {
+	log.SetOutput(ioutil.Discard)
+})

pkg/transformers/dent/model.go

@@ -0,0 +1,25 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dent
+
+type DentModel struct {
+	BidId            string `db:"bid_id"`
+	Lot              string
+	Bid              string
+	Guy              string
+	Tic              string
+	TransactionIndex uint   `db:"tx_idx"`
+	Raw              []byte `db:"raw_log"`
+}

pkg/transformers/dent/repository.go

@@ -0,0 +1,62 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dent
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type Repository interface {
+	Create(headerId int64, model DentModel) error
+	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
+}
+
+type DentRepository struct {
+	db *postgres.DB
+}
+
+func NewDentRepository(database *postgres.DB) DentRepository {
+	return DentRepository{db: database}
+}
+
+func (r DentRepository) Create(headerId int64, model DentModel) error {
+	_, err := r.db.Exec(
+		`INSERT into maker.dent (header_id, bid_id, lot, bid, guy, tic, tx_idx, raw_log)
+         VALUES($1, $2, $3, $4, $5, $6, $7, $8)`,
+		headerId, model.BidId, model.Lot, model.Bid, model.Guy, model.Tic, model.TransactionIndex, model.Raw,
+	)
+
+	return err
+}
+
+func (r DentRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
+	var missingHeaders []core.Header
+
+	err := r.db.Select(
+		&missingHeaders,
+		`SELECT headers.id, headers.block_number FROM headers
+               LEFT JOIN maker.dent on headers.id = header_id
+               WHERE header_id ISNULL
+               AND headers.block_number >= $1
+               AND headers.block_number <= $2
+               AND headers.eth_node_fingerprint = $3`,
+		startingBlockNumber,
+		endingBlockNumber,
+		r.db.Node.ID,
+	)
+
+	return missingHeaders, err
+}

pkg/transformers/dent/repository_test.go

@@ -0,0 +1,153 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dent_test
+
+import (
+	"math/rand"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/dent"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Dent Repository", func() {
+	var node core.Node
+	var db *postgres.DB
+	var dentRepository dent.DentRepository
+	var headerRepository repositories.HeaderRepository
+	var headerId int64
+	var err error
+
+	BeforeEach(func() {
+		node = test_config.NewTestNode()
+		db = test_config.NewTestDB(node)
+		test_config.CleanTestDB(db)
+		dentRepository = dent.NewDentRepository(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+	})
+
+	Describe("Create", func() {
+		BeforeEach(func() {
+			headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+
+			err := dentRepository.Create(headerId, test_data.DentModel)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("persists a dent record", func() {
+			var count int
+			db.QueryRow(`SELECT count(*) FROM maker.dent`).Scan(&count)
+			Expect(count).To(Equal(1))
+
+			var dbResult dent.DentModel
+			err = db.Get(&dbResult, `SELECT bid_id, lot, bid, guy, tic, tx_idx, raw_log FROM maker.dent WHERE header_id = $1`, headerId)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(dbResult.BidId).To(Equal(test_data.DentModel.BidId))
+			Expect(dbResult.Lot).To(Equal(test_data.DentModel.Lot))
+			Expect(dbResult.Bid).To(Equal(test_data.DentModel.Bid))
+			Expect(dbResult.Guy).To(Equal(test_data.DentModel.Guy))
+			Expect(dbResult.Tic).To(Equal(test_data.DentModel.Tic))
+			Expect(dbResult.TransactionIndex).To(Equal(test_data.DentModel.TransactionIndex))
+			Expect(dbResult.Raw).To(MatchJSON(test_data.DentModel.Raw))
+		})
+
+		It("returns an error if inserting a dent record fails", func() {
+			err = dentRepository.Create(headerId, test_data.DentModel)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
+		})
+
+		It("deletes the tend record if its corresponding header record is deleted", func() {
+			var count int
+			err = db.QueryRow(`SELECT count(*) from maker.dent`).Scan(&count)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(count).To(Equal(1))
+
+			_, err = db.Exec(`DELETE FROM headers where id = $1`, headerId)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = db.QueryRow(`SELECT count(*) from maker.dent`).Scan(&count)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(count).To(Equal(0))
+		})
+	})
+
+	Describe("MissingHeaders", func() {
+		var dentBlockNumber int64
+		var startingBlockNumber int64
+		var endingBlockNumber int64
+		var blockNumbers []int64
+
+		BeforeEach(func() {
+			dentBlockNumber = rand.Int63()
+			startingBlockNumber = dentBlockNumber - 1
+			endingBlockNumber = dentBlockNumber + 1
+			outOfRangeBlockNumber := dentBlockNumber + 2
+
+			blockNumbers = []int64{startingBlockNumber, dentBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
+
+			var headerIds []int64
+			for _, number := range blockNumbers {
+				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number})
+				Expect(err).NotTo(HaveOccurred())
+				headerIds = append(headerIds, headerId)
+			}
+
+			dentRepository.Create(headerIds[1], test_data.DentModel)
+		})
+
+		It("returns header records that don't have a corresponding dents", func() {
+			missingHeaders, err := dentRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(missingHeaders)).To(Equal(2))
+			Expect(missingHeaders[0].BlockNumber).To(Equal(startingBlockNumber))
+			Expect(missingHeaders[1].BlockNumber).To(Equal(endingBlockNumber))
+		})
+
+		It("only returns missing headers for the given node", func() {
+			node2 := core.Node{}
+			db2 := test_config.NewTestDB(node2)
+			dentRepository2 := dent.NewDentRepository(db2)
+			headerRepository2 := repositories.NewHeaderRepository(db2)
+			var node2HeaderIds []int64
+			for _, number := range blockNumbers {
+				id, err := headerRepository2.CreateOrUpdateHeader(core.Header{BlockNumber: number})
+				node2HeaderIds = append(node2HeaderIds, id)
+				Expect(err).NotTo(HaveOccurred())
+			}
+
+			missingHeadersNode1, err := dentRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(missingHeadersNode1)).To(Equal(2))
+			Expect(missingHeadersNode1[0].BlockNumber).To(Equal(startingBlockNumber))
+			Expect(missingHeadersNode1[1].BlockNumber).To(Equal(endingBlockNumber))
+
+			missingHeadersNode2, err := dentRepository2.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(missingHeadersNode2)).To(Equal(3))
+			Expect(missingHeadersNode2[0].BlockNumber).To(Equal(startingBlockNumber))
+			Expect(missingHeadersNode2[1].BlockNumber).To(Equal(dentBlockNumber))
+			Expect(missingHeadersNode2[2].BlockNumber).To(Equal(endingBlockNumber))
+		})
+	})
+})

pkg/transformers/dent/transformer.go

@@ -0,0 +1,80 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dent
+
+import (
+	"log"
+
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+)
+
+type DentTransformer struct {
+	Config     shared.TransformerConfig
+	Converter  Converter
+	Fetcher    shared.LogFetcher
+	Repository Repository
+}
+
+type DentTransformerInitializer struct {
+	Config shared.TransformerConfig
+}
+
+func (i DentTransformerInitializer) NewDentTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
+	converter := NewDentConverter()
+	fetcher := shared.NewFetcher(blockChain)
+	repository := NewDentRepository(db)
+	return DentTransformer{
+		Config:     i.Config,
+		Converter:  converter,
+		Fetcher:    fetcher,
+		Repository: repository,
+	}
+}
+
+func (t DentTransformer) Execute() error {
+	config := t.Config
+	topics := [][]common.Hash{{common.HexToHash(shared.DentFunctionSignature)}}
+	headers, err := t.Repository.MissingHeaders(config.StartingBlockNumber, config.EndingBlockNumber)
+	for _, header := range headers {
+		ethLogs, err := t.Fetcher.FetchLogs(config.ContractAddress, topics, header.BlockNumber)
+
+		if err != nil {
+			log.Println("Error fetching dent logs:", err)
+			return err
+		}
+
+		for _, ethLog := range ethLogs {
+			model, err := t.Converter.Convert(config.ContractAddress, config.ContractAbi, ethLog)
+
+			if err != nil {
+				log.Println("Error converting dent log", err)
+				return err
+			}
+
+			err = t.Repository.Create(header.Id, model)
+
+			if err != nil {
+				log.Println("Error persisting dent record", err)
+				return err
+			}
+		}
+	}
+
+	return err
+}

pkg/transformers/dent/transformer_test.go

@@ -0,0 +1,131 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dent_test
+
+import (
+	"math/rand"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/dent"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
+	dent_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/dent"
+)
+
+var _ = Describe("DentTransformer", func() {
+	var config = dent.DentConfig
+	var dentRepository dent_mocks.MockDentRepository
+	var fetcher mocks.MockLogFetcher
+	var converter dent_mocks.MockDentConverter
+	var transformer dent.DentTransformer
+
+	BeforeEach(func() {
+		dentRepository = dent_mocks.MockDentRepository{}
+		fetcher = mocks.MockLogFetcher{}
+		converter = dent_mocks.MockDentConverter{}
+		transformer = dent.DentTransformer{
+			Repository: &dentRepository,
+			Config:     config,
+			Fetcher:    &fetcher,
+			Converter:  &converter,
+		}
+	})
+
+	It("gets missing headers", func() {
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(dentRepository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber))
+		Expect(dentRepository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber))
+	})
+
+	It("returns an error if fetching the missing headers fails", func() {
+		dentRepository.SetMissingHeadersError(fakes.FakeError)
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("fetches logs for each missing header", func() {
+		header1 := core.Header{BlockNumber: rand.Int63()}
+		header2 := core.Header{BlockNumber: rand.Int63()}
+		dentRepository.SetMissingHeaders([]core.Header{header1, header2})
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fetcher.FetchedContractAddress).To(Equal(config.ContractAddress))
+		expectedTopics := [][]common.Hash{{common.HexToHash(shared.DentFunctionSignature)}}
+		Expect(fetcher.FetchedTopics).To(Equal(expectedTopics))
+		Expect(fetcher.FetchedBlocks).To(Equal([]int64{header1.BlockNumber, header2.BlockNumber}))
+	})
+
+	It("returns an error if fetching logs fails", func() {
+		dentRepository.SetMissingHeaders([]core.Header{{}})
+		fetcher.SetFetcherError(fakes.FakeError)
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("converts each eth log to a Model", func() {
+		dentRepository.SetMissingHeaders([]core.Header{{}})
+		fetcher.SetFetchedLogs([]types.Log{test_data.DentLog})
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(converter.PassedContractAddress).To(Equal(config.ContractAddress))
+		Expect(converter.PassedContractAbi).To(Equal(config.ContractAbi))
+		Expect(converter.LogsToConvert).To(Equal([]types.Log{test_data.DentLog}))
+	})
+
+	It("returns an error if converting the eth log fails", func() {
+		dentRepository.SetMissingHeaders([]core.Header{{}})
+		fetcher.SetFetchedLogs([]types.Log{test_data.DentLog})
+		converter.SetConverterError(fakes.FakeError)
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("persists each model as a Dent record", func() {
+		header1 := core.Header{Id: rand.Int63()}
+		header2 := core.Header{Id: rand.Int63()}
+		dentRepository.SetMissingHeaders([]core.Header{header1, header2})
+		fetcher.SetFetchedLogs([]types.Log{test_data.DentLog})
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(dentRepository.PassedDentModels).To(Equal([]dent.DentModel{test_data.DentModel, test_data.DentModel}))
+		Expect(dentRepository.PassedHeaderIds).To(Equal([]int64{header1.Id, header2.Id}))
+	})
+
+	It("returns an error if persisting dent record fails", func() {
+		dentRepository.SetMissingHeaders([]core.Header{{}})
+		dentRepository.SetCreateError(fakes.FakeError)
+		fetcher.SetFetchedLogs([]types.Log{test_data.DentLog})
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+	})
+})

pkg/transformers/flip_kick/config.go

@@ -17,7 +17,7 @@ package flip_kick
 import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 
 var FlipKickConfig = shared.TransformerConfig{
-	ContractAddresses:   "0x08cb6176addcca2e1d1ffe21bee464b72ee4cd8d", //this is a temporary address deployed locally
+	ContractAddress:     shared.FlipperContractAddress,
 	ContractAbi:         shared.FlipperABI,
 	Topics:              []string{shared.FlipKickSignature},
 	StartingBlockNumber: 0,

pkg/transformers/flip_kick/converter.go

@@ -17,7 +17,6 @@ package flip_kick
 import (
 	"encoding/json"
 	"errors"
-	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
@@ -25,7 +24,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 
 	"github.com/vulcanize/vulcanizedb/pkg/geth"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers/utilities"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
 type Converter interface {
@@ -45,7 +44,7 @@ func (FlipKickConverter) ToEntity(contractAddress string, contractAbi string, et
 
 	contract := bind.NewBoundContract(address, abi, nil, nil, nil)
 
-	err = contract.UnpackLog(entity, "FlipKick", ethLog)
+	err = contract.UnpackLog(entity, "Kick", ethLog)
 	if err != nil {
 		return entity, err
 	}
@@ -54,25 +53,18 @@ func (FlipKickConverter) ToEntity(contractAddress string, contractAbi string, et
 }
 
 func (FlipKickConverter) ToModel(flipKick FlipKickEntity) (FlipKickModel, error) {
-	//TODO: Confirm if the following values can be/ever will be nil
-
 	if flipKick.Id == nil {
 		return FlipKickModel{}, errors.New("FlipKick log ID cannot be nil.")
 	}
 
 	id := flipKick.Id.String()
-	vat := strings.ToLower(flipKick.Vat.String())
+	lot := shared.ConvertNilToEmptyString(flipKick.Lot.String())
-	ilk := strings.ToLower(common.ToHex(flipKick.Ilk[:]))
+	bid := shared.ConvertNilToEmptyString(flipKick.Bid.String())
-	lot := utilities.ConvertNilToEmptyString(flipKick.Lot.String())
+	gal := flipKick.Gal.String()
-	bid := utilities.ConvertNilToEmptyString(flipKick.Bid.String())
+	endValue := shared.ConvertNilToZeroTimeValue(flipKick.End)
-	guy := strings.ToLower(flipKick.Guy.String())
-	gal := strings.ToLower(flipKick.Gal.String())
-	endValue := utilities.ConvertNilToZeroTimeValue(flipKick.End)
 	end := time.Unix(endValue, 0)
-	eraValue := utilities.ConvertNilToZeroTimeValue(flipKick.Era)
+	urn := common.BytesToAddress(flipKick.Urn[:common.AddressLength]).String()
-	era := time.Unix(eraValue, 0)
+	tab := shared.ConvertNilToEmptyString(flipKick.Tab.String())
-	lad := strings.ToLower(flipKick.Lad.String())
-	tab := utilities.ConvertNilToEmptyString(flipKick.Tab.String())
 	rawLogJson, err := json.Marshal(flipKick.Raw)
 	if err != nil {
 		return FlipKickModel{}, err
@@ -81,15 +73,11 @@ func (FlipKickConverter) ToModel(flipKick FlipKickEntity) (FlipKickModel, error)
 
 	return FlipKickModel{
 		Id:  id,
-		Vat: vat,
-		Ilk: ilk,
 		Lot: lot,
 		Bid: bid,
-		Guy: guy,
 		Gal: gal,
 		End: end,
-		Era: era,
+		Urn: urn,
-		Lad: lad,
 		Tab: tab,
 		Raw: rawLogString,
 	}, nil

pkg/transformers/flip_kick/converter_test.go

@@ -33,25 +33,21 @@ var _ = Describe("FlipKick Converter", func() {
 
 	Describe("ToEntity", func() {
 		It("converts an Eth Log to a FlipKickEntity", func() {
-			entity, err := converter.ToEntity(test_data.FlipAddress, shared.FlipperABI, test_data.EthFlipKickLog)
+			entity, err := converter.ToEntity(shared.FlipperContractAddress, shared.FlipperABI, test_data.EthFlipKickLog)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(entity.Id).To(Equal(test_data.FlipKickEntity.Id))
-			Expect(entity.Vat).To(Equal(test_data.FlipKickEntity.Vat))
-			Expect(entity.Ilk).To(Equal(test_data.FlipKickEntity.Ilk))
 			Expect(entity.Lot).To(Equal(test_data.FlipKickEntity.Lot))
 			Expect(entity.Bid).To(Equal(test_data.FlipKickEntity.Bid))
-			Expect(entity.Guy).To(Equal(test_data.FlipKickEntity.Guy))
 			Expect(entity.Gal).To(Equal(test_data.FlipKickEntity.Gal))
 			Expect(entity.End).To(Equal(test_data.FlipKickEntity.End))
-			Expect(entity.Era).To(Equal(test_data.FlipKickEntity.Era))
+			Expect(entity.Urn).To(Equal(test_data.FlipKickEntity.Urn))
-			Expect(entity.Lad).To(Equal(test_data.FlipKickEntity.Lad))
 			Expect(entity.Tab).To(Equal(test_data.FlipKickEntity.Tab))
 			Expect(entity.Raw).To(Equal(test_data.FlipKickEntity.Raw))
 		})
 
 		It("returns an error if converting log to entity fails", func() {
-			_, err := converter.ToEntity(test_data.FlipAddress, "error abi", test_data.EthFlipKickLog)
+			_, err := converter.ToEntity(shared.FlipperContractAddress, "error abi", test_data.EthFlipKickLog)
 
 			Expect(err).To(HaveOccurred())
 		})
@@ -59,7 +55,6 @@ var _ = Describe("FlipKick Converter", func() {
 
 	Describe("ToModel", func() {
 		var emptyAddressHex = "0x0000000000000000000000000000000000000000"
-		var emptyByteArrayHex = "0x0000000000000000000000000000000000000000000000000000000000000000"
 		var emptyString = ""
 		var emptyTime = time.Unix(0, 0)
 		var emptyEntity = flip_kick.FlipKickEntity{}
@@ -87,15 +82,11 @@ var _ = Describe("FlipKick Converter", func() {
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(model.Id).To(Equal("1"))
-			Expect(model.Vat).To(Equal(emptyAddressHex))
-			Expect(model.Ilk).To(Equal(emptyByteArrayHex))
 			Expect(model.Lot).To(Equal(emptyString))
 			Expect(model.Bid).To(Equal(emptyString))
-			Expect(model.Guy).To(Equal(emptyAddressHex))
 			Expect(model.Gal).To(Equal(emptyAddressHex))
 			Expect(model.End).To(Equal(emptyTime))
-			Expect(model.Era).To(Equal(emptyTime))
+			Expect(model.Urn).To(Equal(emptyAddressHex))
-			Expect(model.Lad).To(Equal(emptyAddressHex))
 			Expect(model.Tab).To(Equal(emptyString))
 			Expect(model.Raw).To(Equal(emptyRawLog))
 		})

pkg/transformers/flip_kick/entity.go

@@ -23,15 +23,11 @@ import (
 
 type FlipKickEntity struct {
 	Id  *big.Int
-	Vat common.Address
-	Ilk [32]byte
 	Lot *big.Int
 	Bid *big.Int
-	Guy common.Address
 	Gal common.Address
 	End *big.Int
-	Era *big.Int
+	Urn [32]byte
-	Lad common.Address
 	Tab *big.Int
 	Raw types.Log
 }

pkg/transformers/flip_kick/integration_test.go

@@ -17,51 +17,18 @@ package flip_kick_test
 import (
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/ethclient"
-	"github.com/ethereum/go-ethereum/rpc"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/geth"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
-	rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/flip_kick"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
-	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
 var _ = Describe("Integration tests", func() {
-	XIt("Fetches FlipKickEntity event logs from a local test chain", func() {
-		ipcPath := test_config.TestClient.IPCPath
-
-		rawRpcClient, err := rpc.Dial(ipcPath)
-		Expect(err).NotTo(HaveOccurred())
-
-		rpcClient := client.NewRpcClient(rawRpcClient, ipcPath)
-		ethClient := ethclient.NewClient(rawRpcClient)
-		blockChainClient := client.NewEthClient(ethClient)
-		realNode := node.MakeNode(rpcClient)
-		transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-		realBlockChain := geth.NewBlockChain(blockChainClient, realNode, transactionConverter)
-		realFetcher := shared.NewFetcher(realBlockChain)
-		topic0 := common.HexToHash(shared.FlipKickSignature)
-		topics := [][]common.Hash{{topic0}}
-
-		result, err := realFetcher.FetchLogs(test_data.FlipAddress, topics, test_data.FlipKickBlockNumber)
-		Expect(err).NotTo(HaveOccurred())
-
-		Expect(len(result) > 0).To(BeTrue())
-		Expect(result[0].Address).To(Equal(test_data.EthFlipKickLog.Address))
-		Expect(result[0].TxHash).To(Equal(test_data.EthFlipKickLog.TxHash))
-		Expect(result[0].BlockNumber).To(Equal(test_data.EthFlipKickLog.BlockNumber))
-		Expect(result[0].Topics).To(Equal(test_data.EthFlipKickLog.Topics))
-		Expect(result[0].Index).To(Equal(test_data.EthFlipKickLog.Index))
-	})
-
 	It("unpacks an event log", func() {
-		address := common.HexToAddress(test_data.FlipAddress)
+		address := common.HexToAddress(shared.FlipperContractAddress)
 		abi, err := geth.ParseAbi(shared.FlipperABI)
 		Expect(err).NotTo(HaveOccurred())
 
@@ -70,20 +37,16 @@ var _ = Describe("Integration tests", func() {
 
 		var eventLog = test_data.EthFlipKickLog
 
-		err = contract.UnpackLog(entity, "FlipKick", eventLog)
+		err = contract.UnpackLog(entity, "Kick", eventLog)
 		Expect(err).NotTo(HaveOccurred())
 
 		expectedEntity := test_data.FlipKickEntity
 		Expect(entity.Id).To(Equal(expectedEntity.Id))
-		Expect(entity.Vat).To(Equal(expectedEntity.Vat))
-		Expect(entity.Ilk).To(Equal(expectedEntity.Ilk))
 		Expect(entity.Lot).To(Equal(expectedEntity.Lot))
 		Expect(entity.Bid).To(Equal(expectedEntity.Bid))
-		Expect(entity.Guy).To(Equal(expectedEntity.Guy))
 		Expect(entity.Gal).To(Equal(expectedEntity.Gal))
 		Expect(entity.End).To(Equal(expectedEntity.End))
-		Expect(entity.Era).To(Equal(expectedEntity.Era))
+		Expect(entity.Urn).To(Equal(expectedEntity.Urn))
-		Expect(entity.Lad).To(Equal(expectedEntity.Lad))
 		Expect(entity.Tab).To(Equal(expectedEntity.Tab))
 	})
 })

pkg/transformers/flip_kick/model.go

@@ -18,15 +18,11 @@ import "time"
 
 type FlipKickModel struct {
 	Id  string
-	Vat string
-	Ilk string
 	Lot string
 	Bid string
-	Guy string
 	Gal string
 	End time.Time
-	Era time.Time
+	Urn string
-	Lad string
 	Tab string
 	Raw string `db:"raw_log"`
 }

pkg/transformers/flip_kick/repository.go

@@ -35,9 +35,9 @@ func NewFlipKickRepository(db *postgres.DB) FlipKickRepository {
 }
 func (fkr FlipKickRepository) Create(headerId int64, flipKick FlipKickModel) error {
 	_, err := fkr.DB.Exec(
-		`INSERT into maker.flip_kick (header_id, id, vat, ilk, lot, bid, guy, gal, "end", era, lad, tab, raw_log)
+		`INSERT into maker.flip_kick (header_id, id, lot, bid, gal, "end", urn, tab, raw_log)
-        VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)`,
+        VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)`,
-		headerId, flipKick.Id, flipKick.Vat, flipKick.Ilk, flipKick.Lot, flipKick.Bid, flipKick.Guy, flipKick.Gal, flipKick.End, flipKick.Era, flipKick.Lad, flipKick.Tab, flipKick.Raw,
+		headerId, flipKick.Id, flipKick.Lot, flipKick.Bid, flipKick.Gal, flipKick.End, flipKick.Urn, flipKick.Tab, flipKick.Raw,
 	)
 
 	if err != nil {

pkg/transformers/flip_kick/repository_test.go

@@ -65,17 +65,13 @@ var _ = Describe("FlipKick Repository", func() {
 			Expect(err).NotTo(HaveOccurred())
 			Expect(dbResult.HeaderId).To(Equal(headerId))
 			Expect(dbResult.Id).To(Equal(flipKick.Id))
-			Expect(dbResult.Vat).To(Equal(flipKick.Vat))
-			Expect(dbResult.Ilk).To(Equal(flipKick.Ilk))
 			Expect(dbResult.Lot).To(Equal(flipKick.Lot))
 			Expect(dbResult.Bid).To(Equal(flipKick.Bid))
-			Expect(dbResult.Guy).To(Equal(flipKick.Guy))
 			Expect(dbResult.Gal).To(Equal(flipKick.Gal))
 			Expect(dbResult.End.Equal(flipKick.End)).To(BeTrue())
-			Expect(dbResult.Era.Equal(flipKick.Era)).To(BeTrue())
+			Expect(dbResult.Urn).To(Equal(flipKick.Urn))
-			Expect(dbResult.Lad).To(Equal(flipKick.Lad))
 			Expect(dbResult.Tab).To(Equal(flipKick.Tab))
-			Expect(dbResult.Raw).To(Equal(flipKick.Raw))
+			Expect(dbResult.Raw).To(MatchJSON(flipKick.Raw))
 		})
 
 		It("returns an error if inserting the flip_kick record fails", func() {

pkg/transformers/flip_kick/transformer.go

@@ -91,13 +91,13 @@ func (fkt FlipKickTransformer) Execute() error {
 	log.Printf("Fetching event logs for %d headers \n", len(headers))
 	var resultingErrors []error
 	for _, header := range headers {
-		ethLogs, err := fkt.Fetcher.FetchLogs(config.ContractAddresses, topics, header.BlockNumber)
+		ethLogs, err := fkt.Fetcher.FetchLogs(config.ContractAddress, topics, header.BlockNumber)
 		if err != nil {
 			resultingErrors = append(resultingErrors, newTransformerError(err, header.BlockNumber, FetcherError))
 		}
 
 		for _, ethLog := range ethLogs {
-			entity, err := fkt.Converter.ToEntity(config.ContractAddresses, config.ContractAbi, ethLog)
+			entity, err := fkt.Converter.ToEntity(config.ContractAddress, config.ContractAbi, ethLog)
 			if err != nil {
 				resultingErrors = append(resultingErrors, newTransformerError(err, header.BlockNumber, LogToEntityError))
 			}

pkg/transformers/flip_kick/transformer_test.go

@@ -54,7 +54,7 @@ var _ = Describe("FlipKick Transformer", func() {
 
 		startingBlockNumber := rand.Int63()
 		testConfig = shared.TransformerConfig{
-			ContractAddresses:   "0x12345",
+			ContractAddress:     "0x12345",
 			ContractAbi:         "test abi",
 			Topics:              []string{shared.FlipKickSignature},
 			StartingBlockNumber: startingBlockNumber,
@@ -83,7 +83,7 @@ var _ = Describe("FlipKick Transformer", func() {
 		err := transformer.Execute()
 		Expect(err).NotTo(HaveOccurred())
 
-		Expect(fetcher.FetchedContractAddress).To(Equal(testConfig.ContractAddresses))
+		Expect(fetcher.FetchedContractAddress).To(Equal(testConfig.ContractAddress))
 		Expect(fetcher.FetchedTopics).To(Equal(expectedTopics))
 		Expect(fetcher.FetchedBlocks).To(Equal([]int64{blockNumber}))
 	})
@@ -100,7 +100,7 @@ var _ = Describe("FlipKick Transformer", func() {
 		err := transformer.Execute()
 		Expect(err).NotTo(HaveOccurred())
 
-		Expect(converter.ConverterContract).To(Equal(testConfig.ContractAddresses))
+		Expect(converter.ConverterContract).To(Equal(testConfig.ContractAddress))
 		Expect(converter.ConverterAbi).To(Equal(testConfig.ContractAbi))
 		Expect(converter.LogsToConvert).To(Equal(logs))
 		Expect(converter.EntitiesToConvert).To(Equal([]flip_kick.FlipKickEntity{test_data.FlipKickEntity}))

pkg/transformers/frob/config.go

@@ -17,9 +17,9 @@ package frob
 import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 
 var FrobConfig = shared.TransformerConfig{
-	ContractAddresses:   "0xff3f2400f1600f3f493a9a92704a29b96795af1a", //this is a temporary address deployed locally
+	ContractAddress:     shared.PitContractAddress,
-	ContractAbi:         FrobABI,
+	ContractAbi:         shared.PitABI,
-	Topics:              []string{FrobEventSignature},
+	Topics:              []string{shared.FrobSignature},
 	StartingBlockNumber: 0,
 	EndingBlockNumber:   100,
 }

pkg/transformers/frob/constants.go

@@ -1,20 +0,0 @@
-// Copyright 2018 Vulcanize
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package frob
-
-var (
-	FrobABI            = `[{"constant":true,"inputs":[],"name":"vat","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"live","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"drip","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"Line","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"wards","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"bytes32"}],"name":"ilks","outputs":[{"name":"spot","type":"uint256"},{"name":"line","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"inputs":[{"name":"vat_","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"ilk","type":"bytes32"},{"indexed":true,"name":"lad","type":"bytes32"},{"indexed":false,"name":"ink","type":"uint256"},{"indexed":false,"name":"art","type":"uint256"},{"indexed":false,"name":"dink","type":"int256"},{"indexed":false,"name":"dart","type":"int256"},{"indexed":false,"name":"iArt","type":"uint256"}],"name":"Frob","type":"event"},{"anonymous":true,"inputs":[{"indexed":true,"name":"sig","type":"bytes4"},{"indexed":true,"name":"guy","type":"address"},{"indexed":true,"name":"foo","type":"bytes32"},{"indexed":true,"name":"bar","type":"bytes32"},{"indexed":false,"name":"wad","type":"uint256"},{"indexed":false,"name":"fax","type":"bytes"}],"name":"LogNote","type":"event"},{"constant":false,"inputs":[{"name":"guy","type":"address"}],"name":"rely","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"}],"name":"deny","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"ilk","type":"bytes32"},{"name":"what","type":"bytes32"},{"name":"data","type":"uint256"}],"name":"file","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"what","type":"bytes32"},{"name":"data","type":"uint256"}],"name":"file","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"what","type":"bytes32"},{"name":"data","type":"address"}],"name":"file","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"ilk","type":"bytes32"},{"name":"dink","type":"int256"},{"name":"dart","type":"int256"}],"name":"frob","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"}]`
-	FrobEventSignature = "0x6cedf1d3a466a3d6bab04887b1642177bf6dbf1daa737c2e8f639cd0b020d9d0"
-)

pkg/transformers/frob/converter.go

@@ -19,16 +19,16 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 
+	"encoding/json"
 	"github.com/vulcanize/vulcanizedb/pkg/geth"
 )
 
 type Converter interface {
 	ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (FrobEntity, error)
-	ToModel(flipKick FrobEntity) FrobModel
+	ToModel(flipKick FrobEntity) (FrobModel, error)
 }
 
-type FrobConverter struct {
+type FrobConverter struct{}
-}
 
 func (FrobConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (FrobEntity, error) {
 	entity := FrobEntity{}
@@ -39,17 +39,25 @@ func (FrobConverter) ToEntity(contractAddress string, contractAbi string, ethLog
 	}
 	contract := bind.NewBoundContract(address, abi, nil, nil, nil)
 	err = contract.UnpackLog(&entity, "Frob", ethLog)
+	entity.TransactionIndex = ethLog.TxIndex
+	entity.Raw = ethLog
 	return entity, err
 }
 
-func (FrobConverter) ToModel(frob FrobEntity) FrobModel {
+func (FrobConverter) ToModel(frob FrobEntity) (FrobModel, error) {
+	rawLog, err := json.Marshal(frob.Raw)
+	if err != nil {
+		return FrobModel{}, err
+	}
 	return FrobModel{
 		Ilk:              frob.Ilk[:],
-		Lad:  frob.Lad[:],
+		Urn:              frob.Urn[:],
-		Dink: frob.Dink.String(),
-		Dart: frob.Dart.String(),
 		Ink:              frob.Ink.String(),
 		Art:              frob.Art.String(),
+		Dink:             frob.Dink.String(),
+		Dart:             frob.Dart.String(),
 		IArt:             frob.IArt.String(),
-	}
+		TransactionIndex: frob.TransactionIndex,
+		Raw:              rawLog,
+	}, nil
 }

pkg/transformers/frob/converter_test.go

@@ -19,6 +19,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
 )
 
@@ -26,7 +27,7 @@ var _ = Describe("Frob converter", func() {
 	It("converts a log to an entity", func() {
 		converter := frob.FrobConverter{}
 
-		entity, err := converter.ToEntity(test_data.TemporaryFrobAddress, frob.FrobABI, test_data.EthFrobLog)
+		entity, err := converter.ToEntity(shared.PitContractAddress, shared.PitABI, test_data.EthFrobLog)
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(entity).To(Equal(test_data.FrobEntity))
@@ -35,8 +36,9 @@ var _ = Describe("Frob converter", func() {
 	It("converts an entity to a model", func() {
 		converter := frob.FrobConverter{}
 
-		model := converter.ToModel(test_data.FrobEntity)
+		model, err := converter.ToModel(test_data.FrobEntity)
 
+		Expect(err).NotTo(HaveOccurred())
 		Expect(model).To(Equal(test_data.FrobModel))
 	})
 })

pkg/transformers/frob/entity.go

@@ -16,14 +16,18 @@ package frob
 
 import (
 	"math/big"
+
+	"github.com/ethereum/go-ethereum/core/types"
 )
 
 type FrobEntity struct {
 	Ilk              [32]byte
-	Lad  [32]byte
+	Urn              [32]byte
 	Ink              *big.Int
 	Art              *big.Int
 	Dink             *big.Int
 	Dart             *big.Int
 	IArt             *big.Int
+	TransactionIndex uint
+	Raw              types.Log
 }

pkg/transformers/frob/integration_test.go

@@ -46,14 +46,14 @@ var _ = Describe("Integration tests", func() {
 		transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
 		realBlockChain := geth.NewBlockChain(blockChainClient, realNode, transactionConverter)
 		realFetcher := shared.NewFetcher(realBlockChain)
-		topic0 := common.HexToHash(frob.FrobEventSignature)
+		topic0 := common.HexToHash(shared.FrobSignature)
 		topics := [][]common.Hash{{topic0}}
 
-		result, err := realFetcher.FetchLogs(test_data.TemporaryFrobAddress, topics, int64(12))
+		result, err := realFetcher.FetchLogs(shared.PitContractAddress, topics, int64(12))
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(len(result) > 0).To(BeTrue())
-		Expect(result[0].Address).To(Equal(common.HexToAddress(test_data.TemporaryFrobAddress)))
+		Expect(result[0].Address).To(Equal(common.HexToAddress(shared.PitContractAddress)))
 		Expect(result[0].TxHash).To(Equal(test_data.EthFrobLog.TxHash))
 		Expect(result[0].BlockNumber).To(Equal(test_data.EthFrobLog.BlockNumber))
 		Expect(result[0].Topics).To(Equal(test_data.EthFrobLog.Topics))
@@ -61,8 +61,8 @@ var _ = Describe("Integration tests", func() {
 	})
 
 	It("unpacks an event log", func() {
-		address := common.HexToAddress(test_data.TemporaryFrobAddress)
+		address := common.HexToAddress(shared.PitContractAddress)
-		abi, err := geth.ParseAbi(frob.FrobABI)
+		abi, err := geth.ParseAbi(shared.PitABI)
 		Expect(err).NotTo(HaveOccurred())
 
 		contract := bind.NewBoundContract(address, abi, nil, nil, nil)
@@ -78,6 +78,6 @@ var _ = Describe("Integration tests", func() {
 		Expect(entity.IArt).To(Equal(expectedEntity.IArt))
 		Expect(entity.Ilk).To(Equal(expectedEntity.Ilk))
 		Expect(entity.Ink).To(Equal(expectedEntity.Ink))
-		Expect(entity.Lad).To(Equal(expectedEntity.Lad))
+		Expect(entity.Urn).To(Equal(expectedEntity.Urn))
 	})
 })

pkg/transformers/frob/model.go

@@ -16,10 +16,12 @@ package frob
 
 type FrobModel struct {
 	Ilk              []byte
-	Lad  []byte
+	Urn              []byte
 	Ink              string
 	Art              string
 	Dink             string
 	Dart             string
 	IArt             string
+	TransactionIndex uint   `db:"tx_idx"`
+	Raw              []byte `db:"raw_log"`
 }

pkg/transformers/frob/repository.go

@@ -20,7 +20,7 @@ import (
 )
 
 type Repository interface {
-	Create(headerID int64, transactionIndex uint, model FrobModel) error
+	Create(headerID int64, model FrobModel) error
 	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
 }
 
@@ -32,10 +32,10 @@ func NewFrobRepository(db *postgres.DB) FrobRepository {
 	return FrobRepository{db: db}
 }
 
-func (repository FrobRepository) Create(headerID int64, transactionIndex uint, model FrobModel) error {
+func (repository FrobRepository) Create(headerID int64, model FrobModel) error {
-	_, err := repository.db.Exec(`INSERT INTO maker.frob (header_id, tx_idx, art, dart, dink, iart, ilk, ink, lad)
+	_, err := repository.db.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, tx_idx)
-		VALUES($1, $2, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6::NUMERIC, $7, $8::NUMERIC, $9)`,
+		VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10)`,
-		headerID, transactionIndex, model.Art, model.Dart, model.Dink, model.IArt, model.Ilk, model.Ink, model.Lad)
+		headerID, model.Art, model.Dart, model.Dink, model.IArt, model.Ilk, model.Ink, model.Urn, model.Raw, model.TransactionIndex)
 	return err
 }
 

pkg/transformers/frob/repository_test.go

@@ -30,56 +30,61 @@ import (
 var _ = Describe("Frob repository", func() {
 	Describe("Create", func() {
 		It("adds a frob", func() {
-			node := core.Node{}
+			db := test_config.NewTestDB(core.Node{})
-			db := test_config.NewTestDB(node)
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
 			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
 			frobRepository := frob.NewFrobRepository(db)
 
-			err = frobRepository.Create(headerID, 123, test_data.FrobModel)
+			err = frobRepository.Create(headerID, test_data.FrobModel)
 
 			Expect(err).NotTo(HaveOccurred())
 			var dbFrob frob.FrobModel
-			err = db.Get(&dbFrob, `SELECT art, dart, dink, iart, ilk, ink, lad FROM maker.frob WHERE header_id = $1`, headerID)
+			err = db.Get(&dbFrob, `SELECT art, dart, dink, iart, ilk, ink, urn, tx_idx, raw_log FROM maker.frob WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
-			Expect(dbFrob).To(Equal(test_data.FrobModel))
+			Expect(dbFrob.Ilk).To(Equal(test_data.FrobModel.Ilk))
+			Expect(dbFrob.Urn).To(Equal(test_data.FrobModel.Urn))
+			Expect(dbFrob.Ink).To(Equal(test_data.FrobModel.Ink))
+			Expect(dbFrob.Art).To(Equal(test_data.FrobModel.Art))
+			Expect(dbFrob.Dink).To(Equal(test_data.FrobModel.Dink))
+			Expect(dbFrob.Dart).To(Equal(test_data.FrobModel.Dart))
+			Expect(dbFrob.IArt).To(Equal(test_data.FrobModel.IArt))
+			Expect(dbFrob.TransactionIndex).To(Equal(test_data.FrobModel.TransactionIndex))
+			Expect(dbFrob.Raw).To(MatchJSON(test_data.FrobModel.Raw))
 		})
 
 		It("does not duplicate frob events", func() {
-			node := core.Node{}
+			db := test_config.NewTestDB(core.Node{})
-			db := test_config.NewTestDB(node)
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
 			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
 			frobRepository := frob.NewFrobRepository(db)
-			err = frobRepository.Create(headerID, 123, test_data.FrobModel)
+			err = frobRepository.Create(headerID, test_data.FrobModel)
 			Expect(err).NotTo(HaveOccurred())
 
-			err = frobRepository.Create(headerID, 123, test_data.FrobModel)
+			err = frobRepository.Create(headerID, test_data.FrobModel)
 
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("removes frob if corresponding header is deleted", func() {
-			node := core.Node{}
+			db := test_config.NewTestDB(core.Node{})
-			db := test_config.NewTestDB(node)
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
 			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
 			frobRepository := frob.NewFrobRepository(db)
-			err = frobRepository.Create(headerID, 123, test_data.FrobModel)
+			err = frobRepository.Create(headerID, test_data.FrobModel)
 			Expect(err).NotTo(HaveOccurred())
 
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
 			var dbFrob frob.FrobModel
-			err = db.Get(&dbFrob, `SELECT art, iart, ilk, ink, lad FROM maker.frob WHERE header_id = $1`, headerID)
+			err = db.Get(&dbFrob, `SELECT art, iart, ilk, ink, urn, tx_idx, raw_log FROM maker.frob WHERE header_id = $1`, headerID)
 			Expect(err).To(HaveOccurred())
 			Expect(err).To(MatchError(sql.ErrNoRows))
 		})
@@ -87,8 +92,7 @@ var _ = Describe("Frob repository", func() {
 
 	Describe("MissingHeaders", func() {
 		It("returns headers with no associated frob event", func() {
-			node := core.Node{}
+			db := test_config.NewTestDB(core.Node{})
-			db := test_config.NewTestDB(node)
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
 			startingBlockNumber := int64(1)
@@ -102,7 +106,7 @@ var _ = Describe("Frob repository", func() {
 				Expect(err).NotTo(HaveOccurred())
 			}
 			frobRepository := frob.NewFrobRepository(db)
-			err := frobRepository.Create(headerIDs[1], 123, test_data.FrobModel)
+			err := frobRepository.Create(headerIDs[1], test_data.FrobModel)
 			Expect(err).NotTo(HaveOccurred())
 
 			headers, err := frobRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@@ -114,13 +118,11 @@ var _ = Describe("Frob repository", func() {
 		})
 
 		It("only returns headers associated with the current node", func() {
-			nodeOne := core.Node{}
+			db := test_config.NewTestDB(core.Node{})
-			db := test_config.NewTestDB(nodeOne)
 			test_config.CleanTestDB(db)
 			blockNumbers := []int64{1, 2, 3}
 			headerRepository := repositories.NewHeaderRepository(db)
-			nodeTwo := core.Node{ID: "second"}
+			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
-			dbTwo := test_config.NewTestDB(nodeTwo)
 			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
 			var headerIDs []int64
 			for _, n := range blockNumbers {
@@ -132,7 +134,7 @@ var _ = Describe("Frob repository", func() {
 			}
 			frobRepository := frob.NewFrobRepository(db)
 			frobRepositoryTwo := frob.NewFrobRepository(dbTwo)
-			err := frobRepository.Create(headerIDs[0], 0, test_data.FrobModel)
+			err := frobRepository.Create(headerIDs[0], test_data.FrobModel)
 			Expect(err).NotTo(HaveOccurred())
 
 			nodeOneMissingHeaders, err := frobRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

pkg/transformers/frob/transformer.go

@@ -51,18 +51,21 @@ func (transformer FrobTransformer) Execute() error {
 		return err
 	}
 	for _, header := range missingHeaders {
-		topics := [][]common.Hash{{common.HexToHash(FrobEventSignature)}}
+		topics := [][]common.Hash{{common.HexToHash(shared.FrobSignature)}}
-		matchingLogs, err := transformer.Fetcher.FetchLogs(FrobConfig.ContractAddresses, topics, header.BlockNumber)
+		matchingLogs, err := transformer.Fetcher.FetchLogs(FrobConfig.ContractAddress, topics, header.BlockNumber)
 		if err != nil {
 			return err
 		}
 		for _, log := range matchingLogs {
-			entity, err := transformer.Converter.ToEntity(FrobConfig.ContractAddresses, FrobConfig.ContractAbi, log)
+			entity, err := transformer.Converter.ToEntity(FrobConfig.ContractAddress, FrobConfig.ContractAbi, log)
 			if err != nil {
 				return err
 			}
-			model := transformer.Converter.ToModel(entity)
+			model, err := transformer.Converter.ToModel(entity)
-			err = transformer.Repository.Create(header.Id, log.TxIndex, model)
+			if err != nil {
+				return err
+			}
+			err = transformer.Repository.Create(header.Id, model)
 			if err != nil {
 				return err
 			}

pkg/transformers/frob/transformer_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
 	frob_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/frob"
@@ -74,8 +75,8 @@ var _ = Describe("Frob transformer", func() {
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2}))
-		Expect(fetcher.FetchedContractAddress).To(Equal(frob.FrobConfig.ContractAddresses))
+		Expect(fetcher.FetchedContractAddress).To(Equal(frob.FrobConfig.ContractAddress))
-		Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(frob.FrobEventSignature)}}))
+		Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.FrobSignature)}}))
 	})
 
 	It("returns error if fetcher returns error", func() {
@@ -95,7 +96,7 @@ var _ = Describe("Frob transformer", func() {
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
 
-	It("converts matching logs", func() {
+	It("converts matching logs to entity", func() {
 		converter := &frob_mocks.MockFrobConverter{}
 		fetcher := &mocks.MockLogFetcher{}
 		fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
@@ -110,15 +111,51 @@ var _ = Describe("Frob transformer", func() {
 		err := transformer.Execute()
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.PassedContractAddress).To(Equal(frob.FrobConfig.ContractAddresses))
+		Expect(converter.PassedContractAddress).To(Equal(frob.FrobConfig.ContractAddress))
 		Expect(converter.PassedContractABI).To(Equal(frob.FrobConfig.ContractAbi))
 		Expect(converter.PassedLog).To(Equal(test_data.EthFrobLog))
+	})
+
+	It("returns error if converting to entity returns error", func() {
+		converter := &frob_mocks.MockFrobConverter{}
+		converter.SetToEntityError(fakes.FakeError)
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
+		repository := &frob_mocks.MockFrobRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := frob.FrobTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("converts frob entity to model", func() {
+		converter := &frob_mocks.MockFrobConverter{}
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
+		repository := &frob_mocks.MockFrobRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := frob.FrobTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
 		Expect(converter.PassedEntity).To(Equal(test_data.FrobEntity))
 	})
 
-	It("returns error if converter returns error", func() {
+	It("returns error if converting to model returns error", func() {
 		converter := &frob_mocks.MockFrobConverter{}
-		converter.SetConverterError(fakes.FakeError)
+		converter.SetToModelError(fakes.FakeError)
 		fetcher := &mocks.MockLogFetcher{}
 		fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
 		repository := &frob_mocks.MockFrobRepository{}
@@ -152,7 +189,6 @@ var _ = Describe("Frob transformer", func() {
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
-		Expect(repository.PassedTransactionIndex).To(Equal(test_data.EthFrobLog.TxIndex))
 		Expect(repository.PassedFrobModel).To(Equal(test_data.FrobModel))
 	})
 

pkg/transformers/pit_file/config.go

@@ -0,0 +1,27 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pit_file
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+)
+
+var PitFileConfig = shared.TransformerConfig{
+	ContractAddress:     shared.PitContractAddress,
+	ContractAbi:         shared.PitABI,
+	Topics:              []string{shared.PitFileIlkSignature, shared.PitFileDebtCeilingSignature, shared.PitFileStabilityFeeSignature},
+	StartingBlockNumber: 0,
+	EndingBlockNumber:   100,
+}

pkg/transformers/pit_file/debt_ceiling/converter.go

@@ -0,0 +1,59 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package debt_ceiling
+
+import (
+	"encoding/json"
+	"math/big"
+
+	"errors"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+)
+
+type Converter interface {
+	ToModel(ethLog types.Log) (PitFileDebtCeilingModel, error)
+}
+
+type PitFileDebtCeilingConverter struct{}
+
+func (PitFileDebtCeilingConverter) ToModel(ethLog types.Log) (PitFileDebtCeilingModel, error) {
+	err := verifyLog(ethLog)
+	if err != nil {
+		return PitFileDebtCeilingModel{}, err
+	}
+	what := common.HexToAddress(ethLog.Topics[1].String()).String()
+	riskBytes := ethLog.Data[len(ethLog.Data)-shared.DataItemLength:]
+	data := big.NewInt(0).SetBytes(riskBytes).String()
+
+	raw, err := json.Marshal(ethLog)
+	return PitFileDebtCeilingModel{
+		What:             what,
+		Data:             data,
+		TransactionIndex: ethLog.TxIndex,
+		Raw:              raw,
+	}, err
+}
+
+func verifyLog(log types.Log) error {
+	if len(log.Topics) < 2 {
+		return errors.New("log missing topics")
+	}
+	if len(log.Data) < shared.DataItemLength {
+		return errors.New("log missing data")
+	}
+	return nil
+}

pkg/transformers/pit_file/debt_ceiling/converter_test.go

@@ -0,0 +1,58 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package debt_ceiling_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+)
+
+var _ = Describe("", func() {
+	It("returns err if log is missing topics", func() {
+		converter := debt_ceiling.PitFileDebtCeilingConverter{}
+		badLog := types.Log{
+			Data: []byte{1, 1, 1, 1, 1},
+		}
+
+		_, err := converter.ToModel(badLog)
+
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("returns err if log is missing data", func() {
+		converter := debt_ceiling.PitFileDebtCeilingConverter{}
+		badLog := types.Log{
+			Topics: []common.Hash{{}, {}, {}, {}},
+		}
+
+		_, err := converter.ToModel(badLog)
+
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("converts a log to an model", func() {
+		converter := debt_ceiling.PitFileDebtCeilingConverter{}
+
+		model, err := converter.ToModel(test_data.EthPitFileDebtCeilingLog)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(model).To(Equal(test_data.PitFileDebtCeilingModel))
+	})
+})

pkg/transformers/pit_file/debt_ceiling/debt_ceiling_suite_test.go

@@ -0,0 +1,27 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package debt_ceiling_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestDebtCeiling(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "DebtCeiling Suite")
+}

pkg/transformers/pit_file/debt_ceiling/model.go

@@ -0,0 +1,22 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package debt_ceiling
+
+type PitFileDebtCeilingModel struct {
+	What             string
+	Data             string
+	TransactionIndex uint   `db:"tx_idx"`
+	Raw              []byte `db:"raw_log"`
+}

pkg/transformers/pit_file/debt_ceiling/repository.go

@@ -0,0 +1,62 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package debt_ceiling
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type Repository interface {
+	Create(headerID int64, model PitFileDebtCeilingModel) error
+	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
+}
+
+type PitFileDebtCeilingRepository struct {
+	db *postgres.DB
+}
+
+func NewPitFileDebtCeilingRepository(db *postgres.DB) PitFileDebtCeilingRepository {
+	return PitFileDebtCeilingRepository{
+		db: db,
+	}
+}
+
+func (repository PitFileDebtCeilingRepository) Create(headerID int64, model PitFileDebtCeilingModel) error {
+	_, err := repository.db.Exec(
+		`INSERT into maker.pit_file_debt_ceiling (header_id, what, data, tx_idx, raw_log)
+        VALUES($1, $2, $3::NUMERIC, $4, $5)`,
+		headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
+	)
+	return err
+}
+
+func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
+	var result []core.Header
+	err := repository.db.Select(
+		&result,
+		`SELECT headers.id, headers.block_number FROM headers
+               LEFT JOIN maker.pit_file_debt_ceiling on headers.id = header_id
+               WHERE header_id ISNULL
+               AND headers.block_number >= $1
+               AND headers.block_number <= $2
+               AND headers.eth_node_fingerprint = $3`,
+		startingBlockNumber,
+		endingBlockNumber,
+		repository.db.Node.ID,
+	)
+
+	return result, err
+}

pkg/transformers/pit_file/debt_ceiling/repository_test.go

@@ -0,0 +1,144 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package debt_ceiling_test
+
+import (
+	"database/sql"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("", func() {
+	Describe("Create", func() {
+		It("adds a pit file debt ceiling event", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
+
+			err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbPitFile debt_ceiling.PitFileDebtCeilingModel
+			err = db.Get(&dbPitFile, `SELECT what, data, tx_idx, raw_log FROM maker.pit_file_debt_ceiling WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(dbPitFile.What).To(Equal(test_data.PitFileDebtCeilingModel.What))
+			Expect(dbPitFile.Data).To(Equal(test_data.PitFileDebtCeilingModel.Data))
+			Expect(dbPitFile.TransactionIndex).To(Equal(test_data.PitFileDebtCeilingModel.TransactionIndex))
+			Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileDebtCeilingModel.Raw))
+		})
+
+		It("does not duplicate pit file events", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
+			err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
+		})
+
+		It("removes pit file if corresponding header is deleted", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
+			err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbPitFile debt_ceiling.PitFileDebtCeilingModel
+			err = db.Get(&dbPitFile, `SELECT what, data, tx_idx, raw_log FROM maker.pit_file_debt_ceiling WHERE header_id = $1`, headerID)
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(sql.ErrNoRows))
+		})
+	})
+
+	Describe("MissingHeaders", func() {
+		It("returns headers with no associated pit file event", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			startingBlockNumber := int64(1)
+			pitFileBlockNumber := int64(2)
+			endingBlockNumber := int64(3)
+			blockNumbers := []int64{startingBlockNumber, pitFileBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
+			err := pitFileRepository.Create(headerIDs[1], test_data.PitFileDebtCeilingModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(2))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+		})
+
+		It("only returns headers associated with the current node", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			blockNumbers := []int64{1, 2, 3}
+			headerRepository := repositories.NewHeaderRepository(db)
+			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
+			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				Expect(err).NotTo(HaveOccurred())
+			}
+			pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
+			pitFileRepositoryTwo := debt_ceiling.NewPitFileDebtCeilingRepository(dbTwo)
+			err := pitFileRepository.Create(headerIDs[0], test_data.PitFileDebtCeilingModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1))
+
+			nodeTwoMissingHeaders, err := pitFileRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers)))
+		})
+	})
+})

pkg/transformers/pit_file/debt_ceiling/transformer.go

@@ -0,0 +1,72 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package debt_ceiling
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+)
+
+type PitFileDebtCeilingTransformer struct {
+	Config     shared.TransformerConfig
+	Converter  Converter
+	Fetcher    shared.LogFetcher
+	Repository Repository
+}
+
+type PitFileDebtCeilingTransformerInitializer struct {
+	Config shared.TransformerConfig
+}
+
+func (initializer PitFileDebtCeilingTransformerInitializer) NewPitFileDebtCeilingTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
+	converter := PitFileDebtCeilingConverter{}
+	fetcher := shared.NewFetcher(blockChain)
+	repository := NewPitFileDebtCeilingRepository(db)
+	return PitFileDebtCeilingTransformer{
+		Config:     initializer.Config,
+		Converter:  converter,
+		Fetcher:    fetcher,
+		Repository: repository,
+	}
+}
+
+func (transformer PitFileDebtCeilingTransformer) Execute() error {
+	missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber)
+	if err != nil {
+		return err
+	}
+	for _, header := range missingHeaders {
+		topics := [][]common.Hash{{common.HexToHash(shared.PitFileDebtCeilingSignature)}}
+		matchingLogs, err := transformer.Fetcher.FetchLogs(pit_file.PitFileConfig.ContractAddress, topics, header.BlockNumber)
+		if err != nil {
+			return err
+		}
+		for _, log := range matchingLogs {
+			model, err := transformer.Converter.ToModel(log)
+			if err != nil {
+				return err
+			}
+			err = transformer.Repository.Create(header.Id, model)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

pkg/transformers/pit_file/debt_ceiling/transformer_test.go

@@ -0,0 +1,175 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package debt_ceiling_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
+	debt_ceiling_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/pit_file/debt_ceiling"
+)
+
+var _ = Describe("", func() {
+	It("gets missing headers for block numbers specified in config", func() {
+		repository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{}
+		transformer := debt_ceiling.PitFileDebtCeilingTransformer{
+			Config:     pit_file.PitFileConfig,
+			Fetcher:    &mocks.MockLogFetcher{},
+			Converter:  &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.PassedStartingBlockNumber).To(Equal(pit_file.PitFileConfig.StartingBlockNumber))
+		Expect(repository.PassedEndingBlockNumber).To(Equal(pit_file.PitFileConfig.EndingBlockNumber))
+	})
+
+	It("returns error if repository returns error for missing headers", func() {
+		repository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{}
+		repository.SetMissingHeadersErr(fakes.FakeError)
+		transformer := debt_ceiling.PitFileDebtCeilingTransformer{
+			Fetcher:    &mocks.MockLogFetcher{},
+			Converter:  &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("fetches logs for missing headers", func() {
+		fetcher := &mocks.MockLogFetcher{}
+		repository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}})
+		transformer := debt_ceiling.PitFileDebtCeilingTransformer{
+			Fetcher:    fetcher,
+			Converter:  &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2}))
+		Expect(fetcher.FetchedContractAddress).To(Equal(pit_file.PitFileConfig.ContractAddress))
+		Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.PitFileDebtCeilingSignature)}}))
+	})
+
+	It("returns error if fetcher returns error", func() {
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetcherError(fakes.FakeError)
+		repository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := debt_ceiling.PitFileDebtCeilingTransformer{
+			Fetcher:    fetcher,
+			Converter:  &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("converts matching logs", func() {
+		converter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{}
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileDebtCeilingLog})
+		repository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := debt_ceiling.PitFileDebtCeilingTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(converter.PassedLog).To(Equal(test_data.EthPitFileDebtCeilingLog))
+	})
+
+	It("returns error if converter returns error", func() {
+		converter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{}
+		converter.SetConverterError(fakes.FakeError)
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileDebtCeilingLog})
+		repository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := debt_ceiling.PitFileDebtCeilingTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("persists pit file model", func() {
+		converter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{}
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileDebtCeilingLog})
+		repository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{}
+		fakeHeader := core.Header{BlockNumber: 1, Id: 2}
+		repository.SetMissingHeaders([]core.Header{fakeHeader})
+		transformer := debt_ceiling.PitFileDebtCeilingTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
+		Expect(repository.PassedModel).To(Equal(test_data.PitFileDebtCeilingModel))
+	})
+
+	It("returns error if repository returns error for create", func() {
+		converter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{}
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileDebtCeilingLog})
+		repository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}})
+		repository.SetCreateError(fakes.FakeError)
+		transformer := debt_ceiling.PitFileDebtCeilingTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+})

pkg/transformers/pit_file/ilk/converter.go

@@ -0,0 +1,61 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ilk
+
+import (
+	"bytes"
+	"encoding/json"
+	"math/big"
+
+	"errors"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+)
+
+type Converter interface {
+	ToModel(ethLog types.Log) (PitFileIlkModel, error)
+}
+
+type PitFileIlkConverter struct{}
+
+func (PitFileIlkConverter) ToModel(ethLog types.Log) (PitFileIlkModel, error) {
+	err := verifyLog(ethLog)
+	if err != nil {
+		return PitFileIlkModel{}, err
+	}
+	ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+	what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
+	riskBytes := ethLog.Data[len(ethLog.Data)-shared.DataItemLength:]
+	risk := big.NewInt(0).SetBytes(riskBytes).String()
+
+	raw, err := json.Marshal(ethLog)
+	return PitFileIlkModel{
+		Ilk:              ilk,
+		What:             what,
+		Data:             risk,
+		TransactionIndex: ethLog.TxIndex,
+		Raw:              raw,
+	}, err
+}
+
+func verifyLog(log types.Log) error {
+	if len(log.Topics) < 4 {
+		return errors.New("log missing topics")
+	}
+	if len(log.Data) < shared.DataItemLength {
+		return errors.New("log missing data")
+	}
+	return nil
+}

pkg/transformers/pit_file/ilk/converter_test.go

@@ -0,0 +1,58 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ilk_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+)
+
+var _ = Describe("Pit file ilk converter", func() {
+	It("returns err if log is missing topics", func() {
+		converter := ilk.PitFileIlkConverter{}
+		badLog := types.Log{
+			Data: []byte{1, 1, 1, 1, 1},
+		}
+
+		_, err := converter.ToModel(badLog)
+
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("returns err if log is missing data", func() {
+		converter := ilk.PitFileIlkConverter{}
+		badLog := types.Log{
+			Topics: []common.Hash{{}, {}, {}, {}},
+		}
+
+		_, err := converter.ToModel(badLog)
+
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("converts a log to an model", func() {
+		converter := ilk.PitFileIlkConverter{}
+
+		model, err := converter.ToModel(test_data.EthPitFileIlkLog)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(model).To(Equal(test_data.PitFileIlkModel))
+	})
+})

pkg/transformers/pit_file/ilk/ilk_suite_test.go

@@ -0,0 +1,27 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ilk_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestIlk(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Ilk Suite")
+}

pkg/transformers/pit_file/ilk/model.go

@@ -0,0 +1,23 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ilk
+
+type PitFileIlkModel struct {
+	Ilk              string
+	What             string
+	Data             string
+	TransactionIndex uint   `db:"tx_idx"`
+	Raw              []byte `db:"raw_log"`
+}

pkg/transformers/pit_file/ilk/repository.go

@@ -0,0 +1,62 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ilk
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type Repository interface {
+	Create(headerID int64, model PitFileIlkModel) error
+	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
+}
+
+type PitFileIlkRepository struct {
+	db *postgres.DB
+}
+
+func NewPitFileIlkRepository(db *postgres.DB) PitFileIlkRepository {
+	return PitFileIlkRepository{
+		db: db,
+	}
+}
+
+func (repository PitFileIlkRepository) Create(headerID int64, model PitFileIlkModel) error {
+	_, err := repository.db.Exec(
+		`INSERT into maker.pit_file_ilk (header_id, ilk, what, data, tx_idx, raw_log)
+        VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`,
+		headerID, model.Ilk, model.What, model.Data, model.TransactionIndex, model.Raw,
+	)
+	return err
+}
+
+func (repository PitFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
+	var result []core.Header
+	err := repository.db.Select(
+		&result,
+		`SELECT headers.id, headers.block_number FROM headers
+               LEFT JOIN maker.pit_file_ilk on headers.id = header_id
+               WHERE header_id ISNULL
+               AND headers.block_number >= $1
+               AND headers.block_number <= $2
+               AND headers.eth_node_fingerprint = $3`,
+		startingBlockNumber,
+		endingBlockNumber,
+		repository.db.Node.ID,
+	)
+
+	return result, err
+}

pkg/transformers/pit_file/ilk/repository_test.go

@@ -0,0 +1,145 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ilk_test
+
+import (
+	"database/sql"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Pit file ilk repository", func() {
+	Describe("Create", func() {
+		It("adds a pit file ilk event", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			pitFileRepository := ilk.NewPitFileIlkRepository(db)
+
+			err = pitFileRepository.Create(headerID, test_data.PitFileIlkModel)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbPitFile ilk.PitFileIlkModel
+			err = db.Get(&dbPitFile, `SELECT ilk, what, data, tx_idx, raw_log FROM maker.pit_file_ilk WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(dbPitFile.Ilk).To(Equal(test_data.PitFileIlkModel.Ilk))
+			Expect(dbPitFile.What).To(Equal(test_data.PitFileIlkModel.What))
+			Expect(dbPitFile.Data).To(Equal(test_data.PitFileIlkModel.Data))
+			Expect(dbPitFile.TransactionIndex).To(Equal(test_data.PitFileIlkModel.TransactionIndex))
+			Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileIlkModel.Raw))
+		})
+
+		It("does not duplicate pit file ilk events", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			pitFileRepository := ilk.NewPitFileIlkRepository(db)
+			err = pitFileRepository.Create(headerID, test_data.PitFileIlkModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = pitFileRepository.Create(headerID, test_data.PitFileIlkModel)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
+		})
+
+		It("removes pit file ilk if corresponding header is deleted", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			pitFileRepository := ilk.NewPitFileIlkRepository(db)
+			err = pitFileRepository.Create(headerID, test_data.PitFileIlkModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbPitFile ilk.PitFileIlkModel
+			err = db.Get(&dbPitFile, `SELECT ilk, what, data, tx_idx, raw_log FROM maker.pit_file_ilk WHERE header_id = $1`, headerID)
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(sql.ErrNoRows))
+		})
+	})
+
+	Describe("MissingHeaders", func() {
+		It("returns headers with no associated pit file ilk event", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			startingBlockNumber := int64(1)
+			pitFileBlockNumber := int64(2)
+			endingBlockNumber := int64(3)
+			blockNumbers := []int64{startingBlockNumber, pitFileBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			pitFileRepository := ilk.NewPitFileIlkRepository(db)
+			err := pitFileRepository.Create(headerIDs[1], test_data.PitFileIlkModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(2))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+		})
+
+		It("only returns headers associated with the current node", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			blockNumbers := []int64{1, 2, 3}
+			headerRepository := repositories.NewHeaderRepository(db)
+			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
+			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				Expect(err).NotTo(HaveOccurred())
+			}
+			pitFileRepository := ilk.NewPitFileIlkRepository(db)
+			pitFileRepositoryTwo := ilk.NewPitFileIlkRepository(dbTwo)
+			err := pitFileRepository.Create(headerIDs[0], test_data.PitFileIlkModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1))
+
+			nodeTwoMissingHeaders, err := pitFileRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers)))
+		})
+	})
+})

pkg/transformers/pit_file/ilk/transformer.go

@@ -0,0 +1,72 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ilk
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+)
+
+type PitFileIlkTransformer struct {
+	Config     shared.TransformerConfig
+	Converter  Converter
+	Fetcher    shared.LogFetcher
+	Repository Repository
+}
+
+type PitFileIlkTransformerInitializer struct {
+	Config shared.TransformerConfig
+}
+
+func (initializer PitFileIlkTransformerInitializer) NewPitFileIlkTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
+	converter := PitFileIlkConverter{}
+	fetcher := shared.NewFetcher(blockChain)
+	repository := NewPitFileIlkRepository(db)
+	return PitFileIlkTransformer{
+		Config:     initializer.Config,
+		Converter:  converter,
+		Fetcher:    fetcher,
+		Repository: repository,
+	}
+}
+
+func (transformer PitFileIlkTransformer) Execute() error {
+	missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber)
+	if err != nil {
+		return err
+	}
+	for _, header := range missingHeaders {
+		topics := [][]common.Hash{{common.HexToHash(shared.PitFileIlkSignature)}}
+		matchingLogs, err := transformer.Fetcher.FetchLogs(pit_file.PitFileConfig.ContractAddress, topics, header.BlockNumber)
+		if err != nil {
+			return err
+		}
+		for _, log := range matchingLogs {
+			model, err := transformer.Converter.ToModel(log)
+			if err != nil {
+				return err
+			}
+			err = transformer.Repository.Create(header.Id, model)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

pkg/transformers/pit_file/ilk/transformer_test.go

@@ -0,0 +1,175 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ilk_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
+	pit_file_ilk_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/pit_file/ilk"
+)
+
+var _ = Describe("Pit file ilk transformer", func() {
+	It("gets missing headers for block numbers specified in config", func() {
+		repository := &pit_file_ilk_mocks.MockPitFileIlkRepository{}
+		transformer := ilk.PitFileIlkTransformer{
+			Config:     pit_file.PitFileConfig,
+			Fetcher:    &mocks.MockLogFetcher{},
+			Converter:  &pit_file_ilk_mocks.MockPitFileIlkConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.PassedStartingBlockNumber).To(Equal(pit_file.PitFileConfig.StartingBlockNumber))
+		Expect(repository.PassedEndingBlockNumber).To(Equal(pit_file.PitFileConfig.EndingBlockNumber))
+	})
+
+	It("returns error if repository returns error for missing headers", func() {
+		repository := &pit_file_ilk_mocks.MockPitFileIlkRepository{}
+		repository.SetMissingHeadersErr(fakes.FakeError)
+		transformer := ilk.PitFileIlkTransformer{
+			Fetcher:    &mocks.MockLogFetcher{},
+			Converter:  &pit_file_ilk_mocks.MockPitFileIlkConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("fetches logs for missing headers", func() {
+		fetcher := &mocks.MockLogFetcher{}
+		repository := &pit_file_ilk_mocks.MockPitFileIlkRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}})
+		transformer := ilk.PitFileIlkTransformer{
+			Fetcher:    fetcher,
+			Converter:  &pit_file_ilk_mocks.MockPitFileIlkConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2}))
+		Expect(fetcher.FetchedContractAddress).To(Equal(pit_file.PitFileConfig.ContractAddress))
+		Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.PitFileIlkSignature)}}))
+	})
+
+	It("returns error if fetcher returns error", func() {
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetcherError(fakes.FakeError)
+		repository := &pit_file_ilk_mocks.MockPitFileIlkRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := ilk.PitFileIlkTransformer{
+			Fetcher:    fetcher,
+			Converter:  &pit_file_ilk_mocks.MockPitFileIlkConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("converts matching logs", func() {
+		converter := &pit_file_ilk_mocks.MockPitFileIlkConverter{}
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileIlkLog})
+		repository := &pit_file_ilk_mocks.MockPitFileIlkRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := ilk.PitFileIlkTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(converter.PassedLog).To(Equal(test_data.EthPitFileIlkLog))
+	})
+
+	It("returns error if converter returns error", func() {
+		converter := &pit_file_ilk_mocks.MockPitFileIlkConverter{}
+		converter.SetConverterError(fakes.FakeError)
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileIlkLog})
+		repository := &pit_file_ilk_mocks.MockPitFileIlkRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := ilk.PitFileIlkTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("persists pit file model", func() {
+		converter := &pit_file_ilk_mocks.MockPitFileIlkConverter{}
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileIlkLog})
+		repository := &pit_file_ilk_mocks.MockPitFileIlkRepository{}
+		fakeHeader := core.Header{BlockNumber: 1, Id: 2}
+		repository.SetMissingHeaders([]core.Header{fakeHeader})
+		transformer := ilk.PitFileIlkTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
+		Expect(repository.PassedModel).To(Equal(test_data.PitFileIlkModel))
+	})
+
+	It("returns error if repository returns error for create", func() {
+		converter := &pit_file_ilk_mocks.MockPitFileIlkConverter{}
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileIlkLog})
+		repository := &pit_file_ilk_mocks.MockPitFileIlkRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}})
+		repository.SetCreateError(fakes.FakeError)
+		transformer := ilk.PitFileIlkTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+})

pkg/transformers/pit_file/stability_fee/converter.go

@@ -0,0 +1,54 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stability_fee
+
+import (
+	"bytes"
+	"encoding/json"
+
+	"errors"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+type Converter interface {
+	ToModel(ethLog types.Log) (PitFileStabilityFeeModel, error)
+}
+
+type PitFileStabilityFeeConverter struct{}
+
+func (PitFileStabilityFeeConverter) ToModel(ethLog types.Log) (PitFileStabilityFeeModel, error) {
+	err := verifyLog(ethLog)
+	if err != nil {
+		return PitFileStabilityFeeModel{}, err
+	}
+	what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+	data := common.HexToAddress(ethLog.Topics[1].String()).Hex()
+
+	raw, err := json.Marshal(ethLog)
+	return PitFileStabilityFeeModel{
+		What:             what,
+		Data:             data,
+		TransactionIndex: ethLog.TxIndex,
+		Raw:              raw,
+	}, err
+}
+
+func verifyLog(log types.Log) error {
+	if len(log.Topics) < 3 {
+		return errors.New("log missing topics")
+	}
+	return nil
+}

pkg/transformers/pit_file/stability_fee/converter_test.go

@@ -0,0 +1,44 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stability_fee_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/stability_fee"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+)
+
+var _ = Describe("Pit file stability fee converter", func() {
+	It("returns err if log is missing topics", func() {
+		converter := stability_fee.PitFileStabilityFeeConverter{}
+		badLog := types.Log{}
+
+		_, err := converter.ToModel(badLog)
+
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("converts a log to an model", func() {
+		converter := stability_fee.PitFileStabilityFeeConverter{}
+
+		model, err := converter.ToModel(test_data.EthPitFileStabilityFeeLog)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(model).To(Equal(test_data.PitFileStabilityFeeModel))
+	})
+})

pkg/transformers/pit_file/stability_fee/model.go

@@ -0,0 +1,22 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stability_fee
+
+type PitFileStabilityFeeModel struct {
+	What             string
+	Data             string
+	TransactionIndex uint   `db:"tx_idx"`
+	Raw              []byte `db:"raw_log"`
+}

pkg/transformers/pit_file/stability_fee/repository.go

@@ -0,0 +1,62 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stability_fee
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type Repository interface {
+	Create(headerID int64, model PitFileStabilityFeeModel) error
+	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
+}
+
+type PitFileStabilityFeeRepository struct {
+	db *postgres.DB
+}
+
+func NewPitFileStabilityFeeRepository(db *postgres.DB) PitFileStabilityFeeRepository {
+	return PitFileStabilityFeeRepository{
+		db: db,
+	}
+}
+
+func (repository PitFileStabilityFeeRepository) Create(headerID int64, model PitFileStabilityFeeModel) error {
+	_, err := repository.db.Exec(
+		`INSERT into maker.pit_file_stability_fee (header_id, what, data, tx_idx, raw_log)
+        VALUES($1, $2, $3, $4, $5)`,
+		headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
+	)
+	return err
+}
+
+func (repository PitFileStabilityFeeRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
+	var result []core.Header
+	err := repository.db.Select(
+		&result,
+		`SELECT headers.id, headers.block_number FROM headers
+               LEFT JOIN maker.pit_file_stability_fee on headers.id = header_id
+               WHERE header_id ISNULL
+               AND headers.block_number >= $1
+               AND headers.block_number <= $2
+               AND headers.eth_node_fingerprint = $3`,
+		startingBlockNumber,
+		endingBlockNumber,
+		repository.db.Node.ID,
+	)
+
+	return result, err
+}

pkg/transformers/pit_file/stability_fee/repository_test.go

@@ -0,0 +1,144 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stability_fee_test
+
+import (
+	"database/sql"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/stability_fee"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("", func() {
+	Describe("Create", func() {
+		It("adds a pit file stability fee event", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
+
+			err = pitFileRepository.Create(headerID, test_data.PitFileStabilityFeeModel)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbPitFile stability_fee.PitFileStabilityFeeModel
+			err = db.Get(&dbPitFile, `SELECT what, data, tx_idx, raw_log FROM maker.pit_file_stability_fee WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(dbPitFile.What).To(Equal(test_data.PitFileStabilityFeeModel.What))
+			Expect(dbPitFile.Data).To(Equal(test_data.PitFileStabilityFeeModel.Data))
+			Expect(dbPitFile.TransactionIndex).To(Equal(test_data.PitFileStabilityFeeModel.TransactionIndex))
+			Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileStabilityFeeModel.Raw))
+		})
+
+		It("does not duplicate pit file events", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
+			err = pitFileRepository.Create(headerID, test_data.PitFileStabilityFeeModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = pitFileRepository.Create(headerID, test_data.PitFileStabilityFeeModel)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
+		})
+
+		It("removes pit file if corresponding header is deleted", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
+			err = pitFileRepository.Create(headerID, test_data.PitFileStabilityFeeModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbPitFile stability_fee.PitFileStabilityFeeModel
+			err = db.Get(&dbPitFile, `SELECT what, data, tx_idx, raw_log FROM maker.pit_file_stability_fee WHERE header_id = $1`, headerID)
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(sql.ErrNoRows))
+		})
+	})
+
+	Describe("MissingHeaders", func() {
+		It("returns headers with no associated pit file event", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			startingBlockNumber := int64(1)
+			pitFileBlockNumber := int64(2)
+			endingBlockNumber := int64(3)
+			blockNumbers := []int64{startingBlockNumber, pitFileBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
+			err := pitFileRepository.Create(headerIDs[1], test_data.PitFileStabilityFeeModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(2))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+		})
+
+		It("only returns headers associated with the current node", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			blockNumbers := []int64{1, 2, 3}
+			headerRepository := repositories.NewHeaderRepository(db)
+			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
+			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				Expect(err).NotTo(HaveOccurred())
+			}
+			pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
+			pitFileRepositoryTwo := stability_fee.NewPitFileStabilityFeeRepository(dbTwo)
+			err := pitFileRepository.Create(headerIDs[0], test_data.PitFileStabilityFeeModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1))
+
+			nodeTwoMissingHeaders, err := pitFileRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers)))
+		})
+	})
+})

pkg/transformers/pit_file/stability_fee/stability_fee_suite_test.go

@@ -0,0 +1,27 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stability_fee_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestStabilityFee(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "StabilityFee Suite")
+}

pkg/transformers/pit_file/stability_fee/transformer.go

@@ -0,0 +1,72 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stability_fee
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+)
+
+type PitFileStabilityFeeTransformer struct {
+	Config     shared.TransformerConfig
+	Converter  Converter
+	Fetcher    shared.LogFetcher
+	Repository Repository
+}
+
+type PitFileStabilityFeeTransformerInitializer struct {
+	Config shared.TransformerConfig
+}
+
+func (initializer PitFileStabilityFeeTransformerInitializer) NewPitFileStabilityFeeTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
+	converter := PitFileStabilityFeeConverter{}
+	fetcher := shared.NewFetcher(blockChain)
+	repository := NewPitFileStabilityFeeRepository(db)
+	return PitFileStabilityFeeTransformer{
+		Config:     initializer.Config,
+		Converter:  converter,
+		Fetcher:    fetcher,
+		Repository: repository,
+	}
+}
+
+func (transformer PitFileStabilityFeeTransformer) Execute() error {
+	missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber)
+	if err != nil {
+		return err
+	}
+	for _, header := range missingHeaders {
+		topics := [][]common.Hash{{common.HexToHash(shared.PitFileStabilityFeeSignature)}}
+		matchingLogs, err := transformer.Fetcher.FetchLogs(pit_file.PitFileConfig.ContractAddress, topics, header.BlockNumber)
+		if err != nil {
+			return err
+		}
+		for _, log := range matchingLogs {
+			model, err := transformer.Converter.ToModel(log)
+			if err != nil {
+				return err
+			}
+			err = transformer.Repository.Create(header.Id, model)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

pkg/transformers/pit_file/stability_fee/transformer_test.go

@@ -0,0 +1,175 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stability_fee_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/stability_fee"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
+	stability_fee_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/pit_file/stability_fee"
+)
+
+var _ = Describe("", func() {
+	It("gets missing headers for block numbers specified in config", func() {
+		repository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{}
+		transformer := stability_fee.PitFileStabilityFeeTransformer{
+			Config:     pit_file.PitFileConfig,
+			Fetcher:    &mocks.MockLogFetcher{},
+			Converter:  &stability_fee_mocks.MockPitFileStabilityFeeConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.PassedStartingBlockNumber).To(Equal(pit_file.PitFileConfig.StartingBlockNumber))
+		Expect(repository.PassedEndingBlockNumber).To(Equal(pit_file.PitFileConfig.EndingBlockNumber))
+	})
+
+	It("returns error if repository returns error for missing headers", func() {
+		repository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{}
+		repository.SetMissingHeadersErr(fakes.FakeError)
+		transformer := stability_fee.PitFileStabilityFeeTransformer{
+			Fetcher:    &mocks.MockLogFetcher{},
+			Converter:  &stability_fee_mocks.MockPitFileStabilityFeeConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("fetches logs for missing headers", func() {
+		fetcher := &mocks.MockLogFetcher{}
+		repository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}})
+		transformer := stability_fee.PitFileStabilityFeeTransformer{
+			Fetcher:    fetcher,
+			Converter:  &stability_fee_mocks.MockPitFileStabilityFeeConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2}))
+		Expect(fetcher.FetchedContractAddress).To(Equal(pit_file.PitFileConfig.ContractAddress))
+		Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.PitFileStabilityFeeSignature)}}))
+	})
+
+	It("returns error if fetcher returns error", func() {
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetcherError(fakes.FakeError)
+		repository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := stability_fee.PitFileStabilityFeeTransformer{
+			Fetcher:    fetcher,
+			Converter:  &stability_fee_mocks.MockPitFileStabilityFeeConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("converts matching logs", func() {
+		converter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{}
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileStabilityFeeLog})
+		repository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := stability_fee.PitFileStabilityFeeTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(converter.PassedLog).To(Equal(test_data.EthPitFileStabilityFeeLog))
+	})
+
+	It("returns error if converter returns error", func() {
+		converter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{}
+		converter.SetConverterError(fakes.FakeError)
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileStabilityFeeLog})
+		repository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := stability_fee.PitFileStabilityFeeTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("persists pit file model", func() {
+		converter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{}
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileStabilityFeeLog})
+		repository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{}
+		fakeHeader := core.Header{BlockNumber: 1, Id: 2}
+		repository.SetMissingHeaders([]core.Header{fakeHeader})
+		transformer := stability_fee.PitFileStabilityFeeTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
+		Expect(repository.PassedModel).To(Equal(test_data.PitFileStabilityFeeModel))
+	})
+
+	It("returns error if repository returns error for create", func() {
+		converter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{}
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileStabilityFeeLog})
+		repository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}})
+		repository.SetCreateError(fakes.FakeError)
+		transformer := stability_fee.PitFileStabilityFeeTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+})

pkg/transformers/price_feeds/config.go

@@ -14,11 +14,9 @@
 
 package price_feeds
 
-var (
+import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
-	PepAddress = "0x99041F808D598B782D5a3e498681C2452A31da08"
+
-	PipAddress = "0x729D19f657BD0614b4985Cf1D82531c67569197B"
+var ()
-	RepAddress = "0xF5f94b7F9De14D43112e713835BCef2d55b76c1C"
-)
 
 type IPriceFeedConfig struct {
 	ContractAddresses   []string
@@ -28,9 +26,9 @@ type IPriceFeedConfig struct {
 
 var PriceFeedConfig = IPriceFeedConfig{
 	ContractAddresses: []string{
-		PepAddress,
+		shared.PepContractAddress,
-		PipAddress,
+		shared.PipContractAddress,
-		RepAddress,
+		shared.RepContractAddress,
 	},
 	StartingBlockNumber: 0,
 	EndingBlockNumber:   100,

pkg/transformers/price_feeds/constants.go

@@ -22,7 +22,5 @@ import (
 var (
 	ErrNoMatchingLog = errors.New("no matching log")
 	Ether            = big.NewFloat(1e18)
-	LogValueTopic0   = "0x296ba4ca62c6c21c95e828080cb8aec7481b71390585605300a8a76f9e95b527"
-	MedianizerABI    = `[{"constant":false,"inputs":[{"name":"owner_","type":"address"}],"name":"setOwner","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"","type":"bytes32"}],"name":"poke","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[],"name":"poke","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"compute","outputs":[{"name":"","type":"bytes32"},{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"wat","type":"address"}],"name":"set","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"wat","type":"address"}],"name":"unset","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"indexes","outputs":[{"name":"","type":"bytes12"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"next","outputs":[{"name":"","type":"bytes12"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"read","outputs":[{"name":"","type":"bytes32"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"peek","outputs":[{"name":"","type":"bytes32"},{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"bytes12"}],"name":"values","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"min_","type":"uint96"}],"name":"setMin","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"authority_","type":"address"}],"name":"setAuthority","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[],"name":"void","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"pos","type":"bytes12"},{"name":"wat","type":"address"}],"name":"set","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"authority","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"pos","type":"bytes12"}],"name":"unset","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"next_","type":"bytes12"}],"name":"setNext","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"min","outputs":[{"name":"","type":"uint96"}],"payable":false,"stateMutability":"view","type":"function"},{"anonymous":false,"inputs":[{"indexed":false,"name":"val","type":"bytes32"}],"name":"LogValue","type":"event"},{"anonymous":true,"inputs":[{"indexed":true,"name":"sig","type":"bytes4"},{"indexed":true,"name":"guy","type":"address"},{"indexed":true,"name":"foo","type":"bytes32"},{"indexed":true,"name":"bar","type":"bytes32"},{"indexed":false,"name":"wad","type":"uint256"},{"indexed":false,"name":"fax","type":"bytes"}],"name":"LogNote","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"authority","type":"address"}],"name":"LogSetAuthority","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"owner","type":"address"}],"name":"LogSetOwner","type":"event"}]]`
 	Ray              = big.NewFloat(1e27)
 )

pkg/transformers/price_feeds/converter_test.go

@@ -21,6 +21,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
 var _ = Describe("Price feed converter", func() {
@@ -31,7 +32,7 @@ var _ = Describe("Price feed converter", func() {
 		// https://etherscan.io/tx/0xa51a50a2adbfba4e2ab3d72dfd67a21c769f1bc8d2b180663a15500a56cde58f
 		log := types.Log{
 			Address:     medianizerAddress,
-			Topics:      []common.Hash{common.HexToHash(price_feeds.LogValueTopic0)},
+			Topics:      []common.Hash{common.HexToHash(shared.LogValueSignature)},
 			Data:        common.FromHex("00000000000000000000000000000000000000000000001486f658319fb0c100"),
 			BlockNumber: blockNumber,
 			TxHash:      common.HexToHash("0xa51a50a2adbfba4e2ab3d72dfd67a21c769f1bc8d2b180663a15500a56cde58f"),

pkg/transformers/price_feeds/fetcher.go

@@ -15,11 +15,14 @@
 package price_feeds
 
 import (
+	"math/big"
+
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"math/big"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
 type IPriceFeedFetcher interface {
@@ -48,7 +51,7 @@ func (fetcher PriceFeedFetcher) FetchLogValues(blockNumber int64) ([]types.Log,
 		FromBlock: n,
 		ToBlock:   n,
 		Addresses: addresses,
-		Topics:    [][]common.Hash{{common.HexToHash(LogValueTopic0)}},
+		Topics:    [][]common.Hash{{common.HexToHash(shared.LogValueSignature)}},
 	}
 	return fetcher.blockChain.GetEthLogsWithCustomQuery(query)
 }

pkg/transformers/price_feeds/fetcher_test.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
 var _ = Describe("Price fetcher", func() {
@@ -46,7 +47,7 @@ var _ = Describe("Price fetcher", func() {
 			FromBlock: big.NewInt(blockNumber),
 			ToBlock:   big.NewInt(blockNumber),
 			Addresses: expectedAddresses,
-			Topics:    [][]common.Hash{{common.HexToHash(price_feeds.LogValueTopic0)}},
+			Topics:    [][]common.Hash{{common.HexToHash(shared.LogValueSignature)}},
 		}
 		mockBlockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery)
 	})

pkg/transformers/price_feeds/price_update.go

@@ -15,8 +15,9 @@
 package price_feeds
 
 import (
-	"github.com/ethereum/go-ethereum/common"
 	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
 )
 
 type LogValueEntity struct {

pkg/transformers/price_feeds/transformer_test.go

@@ -16,23 +16,25 @@ package price_feeds_test
 
 import (
 	"fmt"
+	"math/big"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
-	price_feeds2 "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/price_feeds"
+	price_feeds_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/price_feeds"
-	"math/big"
 )
 
 var _ = Describe("Price feed transformer", func() {
 	It("gets missing headers for price feeds", func() {
-		mockRepository := &price_feeds2.MockPriceFeedRepository{}
+		mockRepository := &price_feeds_mocks.MockPriceFeedRepository{}
 		transformer := price_feeds.PriceFeedTransformer{
 			Config:     price_feeds.PriceFeedConfig,
-			Fetcher:    &price_feeds2.MockPriceFeedFetcher{},
+			Fetcher:    &price_feeds_mocks.MockPriceFeedFetcher{},
 			Repository: mockRepository,
 		}
 
@@ -43,10 +45,10 @@ var _ = Describe("Price feed transformer", func() {
 	})
 
 	It("returns error is missing headers call returns err", func() {
-		mockRepository := &price_feeds2.MockPriceFeedRepository{}
+		mockRepository := &price_feeds_mocks.MockPriceFeedRepository{}
 		mockRepository.SetMissingHeadersErr(fakes.FakeError)
 		transformer := price_feeds.PriceFeedTransformer{
-			Fetcher:    &price_feeds2.MockPriceFeedFetcher{},
+			Fetcher:    &price_feeds_mocks.MockPriceFeedFetcher{},
 			Repository: mockRepository,
 		}
 
@@ -57,11 +59,11 @@ var _ = Describe("Price feed transformer", func() {
 	})
 
 	It("fetches logs for missing headers", func() {
-		mockRepository := &price_feeds2.MockPriceFeedRepository{}
+		mockRepository := &price_feeds_mocks.MockPriceFeedRepository{}
 		blockNumberOne := int64(1)
 		blockNumberTwo := int64(2)
 		mockRepository.SetMissingHeaders([]core.Header{{BlockNumber: blockNumberOne}, {BlockNumber: blockNumberTwo}})
-		mockFetcher := &price_feeds2.MockPriceFeedFetcher{}
+		mockFetcher := &price_feeds_mocks.MockPriceFeedFetcher{}
 		transformer := price_feeds.PriceFeedTransformer{
 			Fetcher:    mockFetcher,
 			Repository: mockRepository,
@@ -74,9 +76,9 @@ var _ = Describe("Price feed transformer", func() {
 	})
 
 	It("returns err if fetcher returns err", func() {
-		mockRepository := &price_feeds2.MockPriceFeedRepository{}
+		mockRepository := &price_feeds_mocks.MockPriceFeedRepository{}
 		mockRepository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
-		mockFetcher := &price_feeds2.MockPriceFeedFetcher{}
+		mockFetcher := &price_feeds_mocks.MockPriceFeedFetcher{}
 		mockFetcher.SetReturnErr(fakes.FakeError)
 		transformer := price_feeds.PriceFeedTransformer{
 			Fetcher:    mockFetcher,
@@ -90,10 +92,10 @@ var _ = Describe("Price feed transformer", func() {
 	})
 
 	It("persists model converted from log", func() {
-		mockRepository := &price_feeds2.MockPriceFeedRepository{}
+		mockRepository := &price_feeds_mocks.MockPriceFeedRepository{}
 		headerID := int64(11111)
 		mockRepository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: headerID}})
-		mockFetcher := &price_feeds2.MockPriceFeedFetcher{}
+		mockFetcher := &price_feeds_mocks.MockPriceFeedFetcher{}
 		blockNumber := uint64(22222)
 		txIndex := uint(33333)
 		usdValue := int64(44444)
@@ -132,10 +134,10 @@ var _ = Describe("Price feed transformer", func() {
 	})
 
 	It("returns error if creating price feed update returns error", func() {
-		mockRepository := &price_feeds2.MockPriceFeedRepository{}
+		mockRepository := &price_feeds_mocks.MockPriceFeedRepository{}
 		mockRepository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}})
 		mockRepository.SetCreateErr(fakes.FakeError)
-		mockFetcher := &price_feeds2.MockPriceFeedFetcher{}
+		mockFetcher := &price_feeds_mocks.MockPriceFeedFetcher{}
 		mockFetcher.SetReturnLogs([]types.Log{{}})
 		transformer := price_feeds.PriceFeedTransformer{
 			Fetcher:    mockFetcher,

pkg/transformers/shared/event_signature_generator.go

@@ -0,0 +1,56 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shared
+
+import (
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"fmt"
+	"strings"
+)
+
+func GetEventSignature(solidityMethodSignature string) string {
+	eventSignature := []byte(solidityMethodSignature)
+	hash := crypto.Keccak256Hash(eventSignature)
+	return hash.Hex()
+}
+
+func GetLogNoteSignature(solidityMethodSignature string) string {
+	rawSignature := GetEventSignature(solidityMethodSignature)
+	return rawSignature[:10] + "00000000000000000000000000000000000000000000000000000000"
+}
+
+func GetSolidityMethodSignature(abi, name string) string {
+	parsedAbi, _ := geth.ParseAbi(abi)
+
+	if method, ok := parsedAbi.Methods[name]; ok {
+		return method.Sig()
+	} else if event, ok := parsedAbi.Events[name]; ok {
+		return getEventSignature(event)
+	}
+	return ""
+
+}
+
+func getEventSignature(event abi.Event) string {
+	types := make([]string, len(event.Inputs))
+	for i, input := range event.Inputs {
+		types[i] = input.Type.String()
+		i++
+	}
+
+	return fmt.Sprintf("%v(%v)", event.Name, strings.Join(types, ","))
+}

pkg/transformers/shared/event_signature_generator_test.go

@@ -0,0 +1,148 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shared_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+)
+
+var _ = Describe("Event signature generator", func() {
+	Describe("generating non-anonymous event signatures", func() {
+		It("generates bite event signature", func() {
+			expected := "0x99b5620489b6ef926d4518936cfec15d305452712b88bd59da2d9c10fb0953e8"
+			actual := shared.GetEventSignature("Bite(bytes32,bytes32,uint256,uint256,uint256,uint256,uint256)")
+
+			Expect(expected).To(Equal(actual))
+		})
+
+		It("generates frob event signature", func() {
+			expected := "0xb2afa28318bcc689926b52835d844de174ef8de97e982a85c0199d584920791b"
+			actual := shared.GetEventSignature("Frob(bytes32,bytes32,uint256,uint256,int256,int256,uint256)")
+
+			Expect(expected).To(Equal(actual))
+		})
+
+		It("generates flip kick event signature", func() {
+			expected := "0xbac86238bdba81d21995024470425ecb370078fa62b7271b90cf28cbd1e3e87e"
+			actual := shared.GetEventSignature("Kick(uint256,uint256,uint256,address,uint48,bytes32,uint256)")
+
+			Expect(expected).To(Equal(actual))
+		})
+
+		It("generates log value event signature", func() {
+			expected := "0x296ba4ca62c6c21c95e828080cb8aec7481b71390585605300a8a76f9e95b527"
+			actual := shared.GetEventSignature("LogValue(bytes32)")
+
+			Expect(expected).To(Equal(actual))
+		})
+	})
+
+	Describe("generating LogNote event signatures", func() {
+		It("generates flip tend event signature", func() {
+			expected := "0x4b43ed1200000000000000000000000000000000000000000000000000000000"
+			actual := shared.GetLogNoteSignature("tend(uint256,uint256,uint256)")
+
+			Expect(expected).To(Equal(actual))
+		})
+
+		It("generates pit file event signature for overloaded function with three arguments", func() {
+			expected := "0x1a0b287e00000000000000000000000000000000000000000000000000000000"
+			actual := shared.GetLogNoteSignature("file(bytes32,bytes32,uint256)")
+
+			Expect(expected).To(Equal(actual))
+		})
+
+		It("generates pit file event signature for overloaded function with two arguments", func() {
+			expected := "0x29ae811400000000000000000000000000000000000000000000000000000000"
+			actual := shared.GetLogNoteSignature("file(bytes32,uint256)")
+
+			Expect(expected).To(Equal(actual))
+		})
+
+		It("generates pit file event signature for overloaded function with two different arguments", func() {
+			expected := "0xd4e8be8300000000000000000000000000000000000000000000000000000000"
+			actual := shared.GetLogNoteSignature("file(bytes32,address)")
+
+			Expect(expected).To(Equal(actual))
+		})
+	})
+
+	Describe("getting the solidity method/event signature from the abi", func() {
+		Describe("it handles methods", func() {
+			It("gets the flip dent method signature", func() {
+				expected := "dent(uint256,uint256,uint256)"
+				actual := shared.GetSolidityMethodSignature(shared.FlipperABI, "dent")
+
+				Expect(expected).To(Equal(actual))
+			})
+
+			It("gets the flip tend method signature", func() {
+				expected := "tend(uint256,uint256,uint256)"
+				actual := shared.GetSolidityMethodSignature(shared.FlipperABI, "tend")
+
+				Expect(expected).To(Equal(actual))
+			})
+
+			It("gets the pit file deb ceiling method signature", func() {
+				expected := "file(bytes32,address)"
+				actual := shared.GetSolidityMethodSignature(shared.PitABI, "file")
+
+				Expect(expected).To(Equal(actual))
+			})
+
+			It("gets the vat init method signature", func() {
+				expected := "init(bytes32)"
+				actual := shared.GetSolidityMethodSignature(shared.VatABI, "init")
+
+				Expect(expected).To(Equal(actual))
+			})
+
+		})
+
+		Describe("it handles events", func() {
+			It("gets the Bite event signature", func() {
+				expected := "Bite(bytes32,bytes32,uint256,uint256,uint256,uint256,uint256)"
+				actual := shared.GetSolidityMethodSignature(shared.CatABI, "Bite")
+
+				Expect(expected).To(Equal(actual))
+			})
+
+			It("gets the flip Kick event signature", func() {
+				expected := "Kick(uint256,uint256,uint256,address,uint48,bytes32,uint256)"
+				actual := shared.GetSolidityMethodSignature(shared.FlipperABI, "Kick")
+
+				Expect(expected).To(Equal(actual))
+			})
+
+			It("gets the pit frob event signature", func() {
+				expected := "Frob(bytes32,bytes32,uint256,uint256,int256,int256,uint256)"
+				actual := shared.GetSolidityMethodSignature(shared.PitABI, "Frob")
+
+				Expect(expected).To(Equal(actual))
+			})
+
+			It("gets the log value method signature", func() {
+				expected := "LogValue(bytes32)"
+				actual := shared.GetSolidityMethodSignature(shared.MedianizerABI, "LogValue")
+
+				Expect(expected).To(Equal(actual))
+
+			})
+		})
+	})
+})