Merge pull request #54 from 8thlight/VDB-101-vat-fold-transformer

(VDB-101) Add Vat.fold transformer

cmd/continuousLogSync.go

@@ -106,6 +106,7 @@ func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer
 	transformerInitializerMap["tend"] = transformers.TendTransformerInitializer
 	transformerInitializerMap["vatGrab"] = transformers.VatGrabTransformerInitializer
 	transformerInitializerMap["vatInit"] = transformers.VatInitTransformerInitializer
+	transformerInitializerMap["vatFold"] = transformers.VatFoldTransformerInitializer
 	transformerInitializerMap["vatToll"] = transformers.VatTollTransformerInitializer
 	transformerInitializerMap["vatTune"] = transformers.VatTuneTransformerInitializer
 

db/migrations/1538510582_create_vat_fold_table.down.sql

@@ -0,0 +1,3 @@
+DROP TABLE maker.vat_fold;
+ALTER TABLE public.checked_headers
+  DROP COLUMN vat_fold_checked;

db/migrations/1538510582_create_vat_fold_table.up.sql

@@ -0,0 +1,13 @@
+CREATE TABLE maker.vat_fold (
+  id            SERIAL PRIMARY KEY,
+  header_id     INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
+  ilk text,
+  urn text,
+  rate numeric,
+  tx_idx        INTEGER NOT NULL,
+  raw_log       JSONB,
+  UNIQUE (header_id, tx_idx)
+);
+
+ALTER TABLE public.checked_headers
+  ADD COLUMN vat_fold_checked BOOLEAN NOT NULL DEFAULT FALSE;

db/schema.sql

@@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 10.3
+-- Dumped from database version 10.5
--- Dumped by pg_dump version 10.3
+-- Dumped by pg_dump version 10.5
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -697,6 +697,41 @@ CREATE SEQUENCE maker.tend_id_seq
 ALTER SEQUENCE maker.tend_id_seq OWNED BY maker.tend.id;
 
 
+--
+-- Name: vat_fold; Type: TABLE; Schema: maker; Owner: -
+--
+
+CREATE TABLE maker.vat_fold (
+    id integer NOT NULL,
+    header_id integer NOT NULL,
+    ilk text,
+    urn text,
+    rate numeric,
+    tx_idx integer NOT NULL,
+    raw_log jsonb
+);
+
+
+--
+-- Name: vat_fold_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
+--
+
+CREATE SEQUENCE maker.vat_fold_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: vat_fold_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
+--
+
+ALTER SEQUENCE maker.vat_fold_id_seq OWNED BY maker.vat_fold.id;
+
+
 --
 -- Name: vat_grab; Type: TABLE; Schema: maker; Owner: -
 --
@@ -942,6 +977,7 @@ CREATE TABLE public.checked_headers (
     pit_file_ilk_checked boolean DEFAULT false NOT NULL,
     pit_file_stability_fee_checked boolean DEFAULT false NOT NULL,
     vat_init_checked boolean DEFAULT false NOT NULL,
+    vat_fold_checked boolean DEFAULT false NOT NULL,
     vat_toll_checked boolean DEFAULT false NOT NULL,
     vat_tune_checked boolean DEFAULT false NOT NULL,
     vat_grab_checked boolean DEFAULT false NOT NULL
@@ -1391,6 +1427,13 @@ ALTER TABLE ONLY maker.price_feeds ALTER COLUMN id SET DEFAULT nextval('maker.pr
 ALTER TABLE ONLY maker.tend ALTER COLUMN id SET DEFAULT nextval('maker.tend_id_seq'::regclass);
 
 
+--
+-- Name: vat_fold id; Type: DEFAULT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.vat_fold ALTER COLUMN id SET DEFAULT nextval('maker.vat_fold_id_seq'::regclass);
+
+
 --
 -- Name: vat_grab id; Type: DEFAULT; Schema: maker; Owner: -
 --
@@ -1777,6 +1820,22 @@ ALTER TABLE ONLY maker.tend
     ADD CONSTRAINT tend_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: vat_fold vat_fold_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.vat_fold
+    ADD CONSTRAINT vat_fold_header_id_tx_idx_key UNIQUE (header_id, tx_idx);
+
+
+--
+-- Name: vat_fold vat_fold_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.vat_fold
+    ADD CONSTRAINT vat_fold_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: vat_grab vat_grab_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: -
 --
@@ -2139,6 +2198,14 @@ ALTER TABLE ONLY maker.tend
     ADD CONSTRAINT tend_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
 
 
+--
+-- Name: vat_fold vat_fold_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.vat_fold
+    ADD CONSTRAINT vat_fold_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
+
+
 --
 -- Name: vat_grab vat_grab_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: -
 --

pkg/transformers/DOCUMENTATION.md

@@ -1,16 +1,63 @@
-The main goal of creating a transformer is to fetch specific log events from Ethereum, convert/decode them into usable data and then persist them to VulcanizeDB. For Maker there are two main types of log events that we're tracking: custom events that are defined in the contract solidity code, and LogNote events which utilize the [DSNote library](https://github.com/dapphub/ds-note). The transformer process for each of these different log types is the same, except for the converting process, as denoted below.
+# Transformers
 
-To illustrate how to create a custom log event transformer we'll use the Kick event defined in [flop.sol](https://github.com/makerdao/dss/blob/master/src/flop.sol) as an example.
+## Architecture
 
-1. Get an example raw log event either from mainnet (if the contract has already been deployed), from the Kovan testnet, or by deploying the contract to a local chain and emitting the event manually. We will use the example log event to test drive converting the log to a database model.
+Transformers fetch logs from Ethereum, convert/decode them into usable data, and then persist them in postgres.
 
-1. Fetch the logs from the chain based on log event's topic zero.
+A transformer consists of:
-    - The topic zero is based on the keccak-256 hash of the log event's method signature. These are located in `pkg/transformers/shared/constants.go`.
+
+- A fetcher -> Fetches raw logs from the blockchain and encodes them as go datatypes
+- A converter -> Converts this raw data into a human friendly representation suitable for consumption in the API
+- A repository -> Abstracts the database
+
+For Maker, vulcanize will be run in `lightSync` mode, so it will store all headers, and then fetchers pull relevant logs by making RPC calls.
+
+## Event Types
+
+For Maker there are two main types of log events that we're tracking:
+
+1. Custom events that are defined in the contract solidity code.
+1. `LogNote` events which utilize the [DSNote library](https://github.com/dapphub/ds-note).
+1. `Note` events in the `Vat`
+
+The transformer process for each of these different log types is the same, except for the converting process, as denoted below.
+
+## Creating a Transformer
+
+1. Pull an example event (from kovan / ganache etc.)
+1. Add event sig to [`constants.go`](./shared/constants.go)
+1. Write a test for the event sig in [`event_signature_generator_test.go`](./shared/event_signature_generator_test.go)
+1. Create DB table (using [`create_migration`](../../scripts/create_migration))
+1. Create columns in `checked_headers` in the _same_ migration
+1. Add a line to clean the new table `CleanTestDB` (in [`test_config.go`](../../test_config/test_config.go))
+1. Define `model.go`
+1. Create test event in [`test_data`](./test_data)
+1. Write converter + converter tests
+1. Write repository + repository tests
+1. Create converter + repository mocks
+1. Create an config object [`shared.TransformerConfig`](./shared/transformer.go) in `config.go`
+1. Create transformer + transformer tests
+1. Wire up transformer in [`transformers.go`](./transformers.go)
+1. Wire up transformer in [`continuousLogSync.go`](../../cmd/continuousLogSync.go)
+1. Manually trigger an event and check that it gets persisted to postgres
+
+**Fetching Logs**
+
+1. Generate an example raw log event, by either:
+
+   - Pulling the log directly from the Kovan deployment ([constants.go](https://github.com/8thlight/maker-vulcanizedb/blob/master/pkg/transformers/shared/constants.go)).
+   - Deploying the contract to a local chain and emiting the event manually.
+
+1. Fetch the logs from the chain based on the example event's topic zero:
+
+   - The topic zero is based on the keccak-256 hash of the log event's method signature. These are located in [`pkg/transformers/shared/constants.go`](./shared/constants.go).
    - Most transformers use `shared.LogFetcher` to fetch all logs that match the given topic zero for that log event.
    - Since there are multiple price feed contract address that all use the same `LogValue` event, we have a special implementation of a fetcher specifically for price feeds that can query using all of the contract addresses at once, thus only needing to make one call to the blockchain.
 
-1. Convert the raw log into a database model.
+**Coverting logs**
-    - **Converting most custom events** (such as FlopKick)
+
+- **Converting most custom events** (such as FlopKick)
+
   1.  Convert the raw log into a Go struct.
       - We've been using [go-ethereum's abigen tool](https://github.com/ethereum/go-ethereum/tree/master/cmd/abigen) to get the contract's ABI, and a Go struct that represents the event log. We will unpack the raw logs into this struct.
         - To use abigen: `abigen --sol flip.sol --pkg flip --out {/path/to/output_file}`
@@ -29,15 +76,16 @@ To illustrate how to create a custom log event transformer we'll use the Kick ev
             }
         ```
       - Using go-ethereum's `contract.UnpackLog` method we can unpack the raw log into the FlopperKick struct (which we're referring to as the `entity`).
-            - See the `ToEntity` method in `pkg/transformers/flop_kick/converter`.
+        - See the `ToEntity` method in [`pkg/transformers/flop_kick/converter.go`](./flop_kick/converter.go).
-            - The unpack method will not add the `Raw` or `TransactionIndex` values to the entity struct - both of these values are accessible from the raw types.Log.
   1.  Convert the entity into a database model. See the `ToModel` method in `pkg/transformers/flop_kick/converter`.
 
-    - **Converting Price Feed custom events**
+- **Converting Price Feed custom events**
+
   - Price Feed contracts use the [LogNote event](https://github.com/makerdao/medianizer/blob/master/src/medianizer.sol#L23)
   - The LogNote event takes in the value of the price feed as it's sole argument, and does not index it. This means that this value can be taken directly from the log's data, and then properly converted using the `price_feeds.Convert` method (located in the model.go file).
   - Since this conversion from raw log to model includes less fields than some others, we've chosen to convert it directly to the database model, skipping the `ToEntity` step.
-    - **Converting LogNote events** (such as tend)
+
+- **Converting LogNote events** (such as tend)
   - Since LogNote events are a generic structure, they depend on the method signature of the method that is calling them. For example, the `tend` method is called on the [flip.sol contract](https://github.com/makerdao/dss/blob/master/src/flip.sol#L117), and it's method signature looks like this: `tend(uint,uint,uint)`.
     - The first four bytes of the Keccak-256 hashed method signature will be located in `topic[0]` on the log.
     - The message sender will be in `topic[1]`.
@@ -45,26 +93,38 @@ To illustrate how to create a custom log event transformer we'll use the Kick ev
     - The second parameter passed to `tend` will be `topic[3]`.
     - Any additional parameters will be in the log's data field.
     - More detail is located in the [DSNote repo](https://github.com/dapphub/ds-note).
-1. Get all MissingHeaders:
+
-    - Headers are inserted into VulcanizeDB as part of the `lightSync` command. Then for each transformer we check each header for matching logs.
+**Get all MissingHeaders**
-    - The MissingHeaders method queries the `checked_headers` table to see if the header has been checked for the given log type.
+
-1. Persist the log record to VulcanizeDB.
+- Headers are inserted into VulcanizeDB as part of the `lightSync` command. Then for each transformer we check each header for matching logs.
-    - Each event log has it's own table in the database, as well as it's own column in the `checked_headers` table.
+- The MissingHeaders method queries the `checked_headers` table to see if the header has been checked for the given log type.
+
+**Persist the log record to VulcanizeDB**
+
+- Each event log has it's own table in the database, as well as it's own column in the `checked_headers` table.
   - The `checked_headers` table allows us to keep track of which headers have been checked for a given log type.
-    - To create a new migration file: `./scripts/create_migration create_flop_kick`
+- To create a new migration file: `./scripts/create_migration create_flop_kick`
   - See `db/migrations/1536942529_create_flop_kick.up.sql`.
   - The specific log event tables are all created in the `maker`
     schema.
   - There is a one-many association between `headers` and the log
     event tables. This is so that if a header is removed due to a reorg, the associated log event records are also removed.
-    - To run the migrations: `make migrate HOST=local_host PORT=5432 NAME=vulcanize_private`
+- To run the migrations: `make migrate HOST=local_host PORT=5432 NAME=vulcanize_private`
-    - When a new log record is inserted into VulcanizeDB, we also need to make sure to insert a record into the `checked_headers` table for the given log type.
+- When a new log record is inserted into VulcanizeDB, we also need to make sure to insert a record into the `checked_headers` table for the given log type.
-    - We have been using the repository pattern (i.e. wrapping all SQL/ORM invocations in isolated namespaces per table) to interact with the database, see the `Create` method in `pkg/transformers/flop_kick/repository.go`.
+- We have been using the repository pattern (i.e. wrapping all SQL/ORM invocations in isolated namespaces per table) to interact with the database, see the `Create` method in `pkg/transformers/flop_kick/repository.go`.
-1. MarkHeaderChecked:
+
-    - There is a chance that a header does not have a log for the given transformer's log type, and in this instance we also want to record that the header has been "checked" so that we don't continue to query that header over and over.
+**MarkHeaderChecked**
-    - In the transformer we'll make sure to insert a row for the header indicating that it has been checked for the log type that the transformer is responsible for.
+
-1. Wire each component up in the transformer.
+- There is a chance that a header does not have a log for the given transformer's log type, and in this instance we also want to record that the header has been "checked" so that we don't continue to query that header over and over.
-    - We use a TransformerInitializer struct for each transformer so that we can inject ethRPC and postgresDB connections as well as configuration data (including the contract address, block range, etc.) into the transformer. The TransformerInitializer interface is defined in `pkg/transformers/shared/transformer.go`.
+- In the transformer we'll make sure to insert a row for the header indicating that it has been checked for the log type that the transformer is responsible for.
-    - See any of `pkg/transformers/flop_kick/transformer.go`
+
-    - All of the transformers are then initialized in `pkg/transformers/transformers.go` with their configuration.
+**Wire each component up in the transformer**
-    - The transformers can be executed by using the `continuousLogSync` command, which can be configured to run specific transformers or all transformers.
+
+- We use a TransformerInitializer struct for each transformer so that we can inject ethRPC and postgresDB connections as well as configuration data (including the contract address, block range, etc.) into the transformer. The TransformerInitializer interface is defined in `pkg/transformers/shared/transformer.go`.
+- See any of `pkg/transformers/flop_kick/transformer.go`
+- All of the transformers are then initialized in `pkg/transformers/transformers.go` with their configuration.
+- The transformers can be executed by using the `continuousLogSync` command, which can be configured to run specific transformers or all transformers.
+
+## Useful Documents
+
+[Ethereum Event ABI Specification](https://solidity.readthedocs.io/en/develop/abi-spec.html#events)

pkg/transformers/shared/constants.go

@@ -57,6 +57,7 @@ var (
 	tendMethod                = GetSolidityMethodSignature(FlipperABI, "tend")
 	vatGrabMethod             = GetSolidityMethodSignature(VatABI, "grab")
 	vatInitMethod             = GetSolidityMethodSignature(VatABI, "init")
+	vatFoldMethod             = GetSolidityMethodSignature(VatABI, "fold")
 	vatTollMethod             = GetSolidityMethodSignature(VatABI, "toll")
 	vatTuneMethod             = GetSolidityMethodSignature(VatABI, "tune")
 
@@ -80,6 +81,7 @@ var (
 	TendFunctionSignature        = GetLogNoteSignature(tendMethod)
 	VatGrabSignature             = GetLogNoteSignature(vatGrabMethod)
 	VatInitSignature             = GetLogNoteSignature(vatInitMethod)
+	VatFoldSignature             = GetLogNoteSignature(vatFoldMethod)
 	VatTollSignature             = GetLogNoteSignature(vatTollMethod)
 	VatTuneSignature             = GetLogNoteSignature(vatTuneMethod)
 )

pkg/transformers/shared/event_signature_generator_test.go

@@ -119,6 +119,13 @@ var _ = Describe("Event signature generator", func() {
 				Expect(expected).To(Equal(actual))
 			})
 
+			It("gets the vat init method signature", func() {
+				expected := "fold(bytes32,bytes32,int256)"
+				actual := shared.GetSolidityMethodSignature(shared.VatABI, "fold")
+
+				Expect(expected).To(Equal(actual))
+			})
+
 			It("gets the flip deal method signature", func() {
 				expected := "deal(uint256)"
 				actual := shared.GetSolidityMethodSignature(shared.FlipperABI, "deal")

pkg/transformers/test_data/mocks/vat_fold/converter.go

@@ -0,0 +1,36 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vat_fold
+
+import (
+	"github.com/ethereum/go-ethereum/core/types"
+
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold"
+)
+
+type MockVatFoldConverter struct {
+	converterErr error
+	PassedLogs   []types.Log
+}
+
+func (converter *MockVatFoldConverter) ToModels(ethLogs []types.Log) ([]vat_fold.VatFoldModel, error) {
+	converter.PassedLogs = ethLogs
+	return []vat_fold.VatFoldModel{test_data.VatFoldModel}, converter.converterErr
+}
+
+func (converter *MockVatFoldConverter) SetConverterError(e error) {
+	converter.converterErr = e
+}

pkg/transformers/test_data/mocks/vat_fold/repository.go

@@ -0,0 +1,71 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vat_fold
+
+import (
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold"
+)
+
+type MockVatFoldRepository struct {
+	createErr                       error
+	markHeaderCheckedErr            error
+	MarkHeaderCheckedPassedHeaderID int64
+	missingHeaders                  []core.Header
+	missingHeadersErr               error
+	PassedStartingBlockNumber       int64
+	PassedEndingBlockNumber         int64
+	PassedHeaderID                  int64
+	PassedModels                    []vat_fold.VatFoldModel
+}
+
+func (repository *MockVatFoldRepository) Create(headerID int64, models []vat_fold.VatFoldModel) error {
+	repository.PassedHeaderID = headerID
+	repository.PassedModels = models
+	return repository.createErr
+}
+
+func (repository *MockVatFoldRepository) MarkHeaderChecked(headerID int64) error {
+	repository.MarkHeaderCheckedPassedHeaderID = headerID
+	return repository.markHeaderCheckedErr
+}
+
+func (repository *MockVatFoldRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
+	repository.PassedStartingBlockNumber = startingBlockNumber
+	repository.PassedEndingBlockNumber = endingBlockNumber
+	return repository.missingHeaders, repository.missingHeadersErr
+}
+
+func (repository *MockVatFoldRepository) SetMarkHeaderCheckedErr(e error) {
+	repository.markHeaderCheckedErr = e
+}
+
+func (repository *MockVatFoldRepository) SetMissingHeadersErr(e error) {
+	repository.missingHeadersErr = e
+}
+
+func (repository *MockVatFoldRepository) SetMissingHeaders(headers []core.Header) {
+	repository.missingHeaders = headers
+}
+
+func (repository *MockVatFoldRepository) SetCreateError(e error) {
+	repository.createErr = e
+}
+
+func (repository *MockVatFoldRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
+	Expect(repository.MarkHeaderCheckedPassedHeaderID).To(Equal(i))
+}

pkg/transformers/test_data/vat_fold.go

@@ -0,0 +1,52 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test_data
+
+import (
+	"encoding/json"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold"
+)
+
+var EthVatFoldLog = types.Log{
+	Address: common.HexToAddress(shared.VatContractAddress),
+	Topics: []common.Hash{
+		common.HexToHash("0xe6a6a64d00000000000000000000000000000000000000000000000000000000"),
+		common.HexToHash("0x5245500000000000000000000000000000000000000000000000000000000000"),
+		common.HexToHash("0x0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1"),
+		common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000002"),
+	},
+	Data:        hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000064e6a6a64d45544800000000000000000000000000000000000000000000000000000000000000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d10000000000000000000000000000000000000000000000000000000000000000"),
+	BlockNumber: 8940380,
+	TxHash:      common.HexToHash("0xfb37b7a88aa8ad14538d1e244a55939fa07c1828e5ca8168bf4edd56f5fc4d57"),
+	TxIndex:     8,
+	BlockHash:   common.HexToHash("0xf43ab2fd3cf0a7e08fcc16ec17bbc7f67417a37a4cd978d1d7ca32130c7f64be"),
+	Index:       5,
+	Removed:     false,
+}
+
+var rawVatFoldLog, _ = json.Marshal(EthVatFoldLog)
+var VatFoldModel = vat_fold.VatFoldModel{
+	Ilk:              "REP",
+	Urn:              "0x3728e9777B2a0a611ee0F89e00E01044ce4736d1",
+	Rate:             "2",
+	TransactionIndex: EthVatFoldLog.TxIndex,
+	Raw:              rawVatFoldLog,
+}

pkg/transformers/transformers.go

@@ -37,6 +37,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/tend"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_init"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_toll"
@@ -67,6 +68,7 @@ var (
 	TendTransformerInitializer                = tend.TendTransformerInitializer{Config: tend.TendConfig}.NewTendTransformer
 	VatGrabTransformerInitializer             = vat_grab.VatGrabTransformerInitializer{Config: vat_grab.VatGrabConfig}.NewVatGrabTransformer
 	VatInitTransformerInitializer             = vat_init.VatInitTransformerInitializer{Config: vat_init.VatInitConfig}.NewVatInitTransformer
+	VatFoldTransformerInitializer             = vat_fold.VatFoldTransformerInitializer{Config: vat_fold.VatFoldConfig}.NewVatFoldTransformer
 	VatTollTransformerInitializer             = vat_toll.VatTollTransformerInitializer{Config: vat_toll.VatTollConfig}.NewVatTollTransformer
 	VatTuneTransformerInitializer             = vat_tune.VatTuneTransformerInitializer{Config: vat_tune.VatTuneConfig}.NewVatTuneTransformer
 )
@@ -93,6 +95,7 @@ func TransformerInitializers() []shared.TransformerInitializer {
 		TendTransformerInitializer,
 		VatGrabTransformerInitializer,
 		VatInitTransformerInitializer,
+		VatFoldTransformerInitializer,
 		VatTollTransformerInitializer,
 		VatTuneTransformerInitializer,
 	}

pkg/transformers/vat_fold/config.go

@@ -0,0 +1,27 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vat_fold
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+)
+
+var VatFoldConfig = shared.TransformerConfig{
+	ContractAddresses:   []string{shared.VatContractAddress},
+	ContractAbi:         shared.VatABI,
+	Topics:              []string{shared.VatFoldSignature},
+	StartingBlockNumber: 0,
+	EndingBlockNumber:   10000000,
+}

pkg/transformers/vat_fold/converter.go

@@ -0,0 +1,75 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vat_fold
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+)
+
+type Converter interface {
+	ToModels(ethLogs []types.Log) ([]VatFoldModel, error)
+}
+
+type VatFoldConverter struct{}
+
+func (VatFoldConverter) ToModels(ethLogs []types.Log) ([]VatFoldModel, error) {
+	var models []VatFoldModel
+	for _, ethLog := range ethLogs {
+		err := verifyLog(ethLog)
+		if err != nil {
+			return nil, err
+		}
+
+		ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
+		urn := common.BytesToAddress(ethLog.Topics[2].Bytes()).String()
+		rate := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String()
+		raw, err := json.Marshal(ethLog)
+
+		if err != nil {
+			return models, err
+		}
+
+		model := VatFoldModel{
+			Ilk:              ilk,
+			Urn:              urn,
+			Rate:             rate,
+			TransactionIndex: ethLog.TxIndex,
+			Raw:              raw,
+		}
+
+		models = append(models, model)
+	}
+	return models, nil
+}
+
+func verifyLog(log types.Log) error {
+	if len(log.Topics) < 4 {
+		return errors.New("log missing topics")
+	}
+
+	sig := log.Topics[0].String()
+	if sig != shared.VatFoldSignature {
+		return errors.New("log is not a Vat.fold event")
+	}
+
+	return nil
+}

pkg/transformers/vat_fold/converter_test.go

@@ -0,0 +1,44 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vat_fold_test
+
+import (
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold"
+)
+
+var _ = Describe("Vat fold converter", func() {
+	It("returns err if log missing topics", func() {
+		converter := vat_fold.VatFoldConverter{}
+		badLog := types.Log{}
+
+		_, err := converter.ToModels([]types.Log{badLog})
+
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("converts a log to an model", func() {
+		converter := vat_fold.VatFoldConverter{}
+
+		model, err := converter.ToModels([]types.Log{test_data.EthVatFoldLog})
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(model).To(Equal([]vat_fold.VatFoldModel{test_data.VatFoldModel}))
+	})
+})

pkg/transformers/vat_fold/model.go

@@ -0,0 +1,9 @@
+package vat_fold
+
+type VatFoldModel struct {
+	Ilk              string
+	Urn              string
+	Rate             string
+	TransactionIndex uint   `db:"tx_idx"`
+	Raw              []byte `db:"raw_log"`
+}

pkg/transformers/vat_fold/repository.go

@@ -0,0 +1,95 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vat_fold
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type Repository interface {
+	Create(headerID int64, models []VatFoldModel) error
+	MarkHeaderChecked(headerID int64) error
+	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
+}
+
+type VatFoldRepository struct {
+	DB *postgres.DB
+}
+
+func NewVatFoldRepository(db *postgres.DB) VatFoldRepository {
+	return VatFoldRepository{
+		DB: db,
+	}
+}
+
+func (repository VatFoldRepository) Create(headerID int64, models []VatFoldModel) error {
+	tx, err := repository.DB.Begin()
+	if err != nil {
+		return err
+	}
+	for _, model := range models {
+		_, err = tx.Exec(
+			`INSERT into maker.vat_fold (header_id, ilk, urn, rate, tx_idx, raw_log)
+				VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`,
+			headerID, model.Ilk, model.Urn, model.Rate, model.TransactionIndex, model.Raw,
+		)
+		if err != nil {
+			tx.Rollback()
+			return err
+		}
+		_, err = tx.Exec(
+			`INSERT INTO public.checked_headers (header_id, vat_fold_checked) 
+				VALUES($1, $2)
+			ON CONFLICT (header_id) DO 
+				UPDATE SET vat_fold_checked = $2`,
+			headerID, true,
+		)
+		if err != nil {
+			tx.Rollback()
+			return err
+		}
+	}
+
+	return tx.Commit()
+}
+
+func (repository VatFoldRepository) MarkHeaderChecked(headerID int64) error {
+	_, err := repository.DB.Exec(
+		`INSERT INTO public.checked_headers (header_id, vat_fold_checked)
+			VALUES ($1, $2) 
+		ON CONFLICT (header_id) DO
+			UPDATE SET vat_fold_checked = $2`,
+		headerID, true,
+	)
+	return err
+}
+
+func (repository VatFoldRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
+	var result []core.Header
+	err := repository.DB.Select(
+		&result,
+		`SELECT headers.id, headers.block_number FROM headers
+               LEFT JOIN checked_headers on headers.id = header_id
+               WHERE (header_id ISNULL OR vat_fold_checked IS FALSE)
+               AND headers.block_number >= $1
+               AND headers.block_number <= $2
+               AND headers.eth_node_fingerprint = $3`,
+		startingBlockNumber,
+		endingBlockNumber,
+		repository.DB.Node.ID,
+	)
+	return result, err
+}

pkg/transformers/vat_fold/repository_test.go

@@ -0,0 +1,201 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vat_fold_test
+
+import (
+	"database/sql"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Vat.fold repository", func() {
+
+	var db *postgres.DB
+	var headerID int64
+	var repository vat_fold.VatFoldRepository
+	var headerRepository repositories.HeaderRepository
+	var err error
+
+	BeforeEach(func() {
+		node := test_config.NewTestNode()
+		db = test_config.NewTestDB(node)
+		test_config.CleanTestDB(db)
+		repository = vat_fold.NewVatFoldRepository(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+		headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	Describe("Create", func() {
+
+		BeforeEach(func() {
+			err = repository.Create(headerID, []vat_fold.VatFoldModel{test_data.VatFoldModel})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("adds a vat event", func() {
+			var dbVatFold vat_fold.VatFoldModel
+			err := db.Get(&dbVatFold, `SELECT ilk, urn, rate, tx_idx, raw_log FROM maker.vat_fold WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+
+			Expect(dbVatFold.Ilk).To(Equal(test_data.VatFoldModel.Ilk))
+			Expect(dbVatFold.Urn).To(Equal(test_data.VatFoldModel.Urn))
+			Expect(dbVatFold.Rate).To(Equal(test_data.VatFoldModel.Rate))
+			Expect(dbVatFold.TransactionIndex).To(Equal(test_data.VatFoldModel.TransactionIndex))
+			Expect(dbVatFold.Raw).To(MatchJSON(test_data.VatFoldModel.Raw))
+		})
+
+		It("does not duplicate vat events", func() {
+			err := repository.Create(headerID, []vat_fold.VatFoldModel{test_data.VatFoldModel})
+
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
+		})
+
+		It("removes vat if corresponding header is deleted", func() {
+			_, err := db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+
+			var dbVatFold vat_fold.VatFoldModel
+			err = db.Get(&dbVatFold, `SELECT ilk, tx_idx, raw_log FROM maker.vat_fold WHERE header_id = $1`, headerID)
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(sql.ErrNoRows))
+		})
+	})
+
+	Describe("MarkHeaderChecked", func() {
+		type CheckedHeaderResult struct {
+			VatFoldChecked bool `db:"vat_fold_checked"`
+		}
+
+		It("creates a new checked header record", func() {
+			err := repository.MarkHeaderChecked(headerID)
+			Expect(err).NotTo(HaveOccurred())
+
+			var checkedHeaderResult = CheckedHeaderResult{}
+			err = db.Get(&checkedHeaderResult, `SELECT vat_fold_checked FROM checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(checkedHeaderResult.VatFoldChecked).To(BeTrue())
+		})
+
+		It("updates an existing checked header", func() {
+			_, err := repository.DB.Exec(`INSERT INTO checked_headers (header_id) VALUES($1)`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+
+			var checkedHeaderResult CheckedHeaderResult
+			err = db.Get(&checkedHeaderResult, `SELECT vat_fold_checked FROM checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(checkedHeaderResult.VatFoldChecked).To(BeFalse())
+
+			err = repository.MarkHeaderChecked(headerID)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = db.Get(&checkedHeaderResult, `SELECT vat_fold_checked FROM checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(checkedHeaderResult.VatFoldChecked).To(BeTrue())
+		})
+	})
+
+	Describe("MissingHeaders", func() {
+
+		It("returns headers that haven't been checked", func() {
+			startingBlockNumber := int64(1)
+			vatGrabBlockNumber := int64(2)
+			endingBlockNumber := int64(3)
+			blockNumbers := []int64{startingBlockNumber, vatGrabBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+
+			err := repository.MarkHeaderChecked(headerIDs[1])
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(2))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+		})
+
+		It("only treats headers as checked if vat fold logs have been checked", func() {
+			startingBlockNumber := int64(1)
+			vatGrabdBlockNumber := int64(2)
+			endingBlockNumber := int64(3)
+			blockNumbers := []int64{startingBlockNumber, vatGrabdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+
+			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(3))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatGrabdBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatGrabdBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatGrabdBlockNumber)))
+		})
+
+		It("only returns headers associated with the current node", func() {
+			blockNumbers := []int64{1, 2, 3}
+			headerRepository := repositories.NewHeaderRepository(db)
+
+			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
+			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
+
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
+
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				Expect(err).NotTo(HaveOccurred())
+			}
+
+			repositoryTwo := vat_fold.NewVatFoldRepository(dbTwo)
+
+			err := repository.MarkHeaderChecked(headerIDs[0])
+			Expect(err).NotTo(HaveOccurred())
+
+			nodeOneMissingHeaders, err := repository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1))
+
+			nodeTwoMissingHeaders, err := repositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers)))
+		})
+	})
+})

pkg/transformers/vat_fold/transformer.go

@@ -0,0 +1,79 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vat_fold
+
+import (
+	"log"
+
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+)
+
+type VatFoldTransformerInitializer struct {
+	Config shared.TransformerConfig
+}
+
+func (initializer VatFoldTransformerInitializer) NewVatFoldTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
+	converter := VatFoldConverter{}
+	fetcher := shared.NewFetcher(blockChain)
+	repository := NewVatFoldRepository(db)
+	return VatFoldTransformer{
+		Config:     initializer.Config,
+		Converter:  converter,
+		Fetcher:    fetcher,
+		Repository: repository,
+	}
+}
+
+type VatFoldTransformer struct {
+	Config     shared.TransformerConfig
+	Converter  Converter
+	Fetcher    shared.LogFetcher
+	Repository Repository
+}
+
+func (transformer VatFoldTransformer) Execute() error {
+	missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber)
+	if err != nil {
+		return err
+	}
+	log.Printf("Fetching vat fold event logs for %d headers \n", len(missingHeaders))
+	for _, header := range missingHeaders {
+		topics := [][]common.Hash{{common.HexToHash(shared.VatFoldSignature)}}
+		matchingLogs, err := transformer.Fetcher.FetchLogs(VatFoldConfig.ContractAddresses, topics, header.BlockNumber)
+		if err != nil {
+			return err
+		}
+		if len(matchingLogs) < 1 {
+			err = transformer.Repository.MarkHeaderChecked(header.Id)
+			if err != nil {
+				return err
+			}
+			continue
+		}
+		models, err := transformer.Converter.ToModels(matchingLogs)
+		if err != nil {
+			return err
+		}
+		err = transformer.Repository.Create(header.Id, models)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/transformers/vat_fold/transformer_test.go

@@ -0,0 +1,215 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vat_fold_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
+	vat_fold_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/vat_fold"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold"
+)
+
+type setupOptions struct {
+	setMissingHeadersError bool
+	setFetcherError        bool
+	setConverterError      bool
+	setCreateError         bool
+	fetchedLogs            []types.Log
+	missingHeaders         []core.Header
+}
+
+func setup(options setupOptions) (
+	vat_fold.VatFoldTransformer,
+	*mocks.MockLogFetcher,
+	*vat_fold_mocks.MockVatFoldConverter,
+	*vat_fold_mocks.MockVatFoldRepository,
+) {
+	fetcher := &mocks.MockLogFetcher{}
+	if options.setFetcherError {
+		fetcher.SetFetcherError(fakes.FakeError)
+	}
+	if len(options.fetchedLogs) > 0 {
+		fetcher.SetFetchedLogs(options.fetchedLogs)
+	}
+
+	converter := &vat_fold_mocks.MockVatFoldConverter{}
+	if options.setConverterError {
+		converter.SetConverterError(fakes.FakeError)
+	}
+
+	repository := &vat_fold_mocks.MockVatFoldRepository{}
+	if options.setMissingHeadersError {
+		repository.SetMissingHeadersErr(fakes.FakeError)
+	}
+	if options.setCreateError {
+		repository.SetCreateError(fakes.FakeError)
+	}
+	if len(options.missingHeaders) > 0 {
+		repository.SetMissingHeaders(options.missingHeaders)
+	}
+
+	transformer := vat_fold.VatFoldTransformer{
+		Config:     vat_fold.VatFoldConfig,
+		Fetcher:    fetcher,
+		Converter:  converter,
+		Repository: repository,
+	}
+
+	return transformer, fetcher, converter, repository
+}
+
+var _ = Describe("Vat fold transformer", func() {
+	It("gets missing headers for block numbers specified in config", func() {
+		transformer, _, _, repository := setup(setupOptions{})
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.PassedStartingBlockNumber).To(Equal(vat_fold.VatFoldConfig.StartingBlockNumber))
+		Expect(repository.PassedEndingBlockNumber).To(Equal(vat_fold.VatFoldConfig.EndingBlockNumber))
+	})
+
+	It("returns error if repository returns error for missing headers", func() {
+		transformer, _, _, _ := setup(setupOptions{
+			setMissingHeadersError: true,
+		})
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("fetches logs for missing headers", func() {
+		transformer, fetcher, _, _ := setup(setupOptions{
+			missingHeaders: []core.Header{{BlockNumber: 1}, {BlockNumber: 2}},
+		})
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2}))
+		Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{vat_fold.VatFoldConfig.ContractAddresses, vat_fold.VatFoldConfig.ContractAddresses}))
+		Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.VatFoldSignature)}}))
+	})
+
+	It("returns error if fetcher returns error", func() {
+		transformer, _, _, _ := setup(setupOptions{
+			setFetcherError: true,
+			missingHeaders:  []core.Header{{BlockNumber: 1}},
+		})
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("converts matching logs", func() {
+		transformer, _, converter, _ := setup(setupOptions{
+			fetchedLogs:    []types.Log{test_data.EthVatFoldLog},
+			missingHeaders: []core.Header{{BlockNumber: 1}},
+		})
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthVatFoldLog}))
+	})
+
+	It("returns error if converter returns error", func() {
+		transformer, _, _, _ := setup(setupOptions{
+			setConverterError: true,
+			fetchedLogs:       []types.Log{test_data.EthVatFoldLog},
+			missingHeaders:    []core.Header{{BlockNumber: 1}},
+		})
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("persists vat fold model", func() {
+		fakeHeader := core.Header{BlockNumber: 1, Id: 2}
+		transformer, _, _, repository := setup(setupOptions{
+			fetchedLogs:    []types.Log{test_data.EthVatFoldLog},
+			missingHeaders: []core.Header{fakeHeader},
+		})
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
+		Expect(repository.PassedModels).To(Equal([]vat_fold.VatFoldModel{test_data.VatFoldModel}))
+	})
+
+	It("returns error if repository returns error for create", func() {
+		transformer, _, _, _ := setup(setupOptions{
+			fetchedLogs:    []types.Log{test_data.EthVatFoldLog},
+			missingHeaders: []core.Header{{BlockNumber: 1, Id: 2}},
+			setCreateError: true,
+		})
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("marks the header as checked when there are no logs", func() {
+		header := core.Header{Id: GinkgoRandomSeed()}
+		transformer, _, _, repository := setup(setupOptions{
+			missingHeaders: []core.Header{header},
+		})
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.MarkHeaderCheckedPassedHeaderID).To(Equal(header.Id))
+	})
+
+	It("doesn't call MarkHeaderChecked when there are logs", func() {
+		transformer, _, _, repository := setup(setupOptions{
+			missingHeaders: []core.Header{{Id: GinkgoRandomSeed()}},
+			fetchedLogs:    []types.Log{test_data.EthVatFoldLog},
+		})
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.MarkHeaderCheckedPassedHeaderID).To(Equal(int64(0)))
+	})
+
+	It("returns an error if MarkHeaderChecked fails", func() {
+		transformer, _, _, _ := setup(setupOptions{
+			missingHeaders:         []core.Header{{Id: GinkgoRandomSeed()}},
+			setMissingHeadersError: true,
+		})
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+})

pkg/transformers/vat_fold/vat_fold_suite_test.go

@@ -0,0 +1,19 @@
+package vat_fold_test
+
+import (
+	"io/ioutil"
+	"log"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestVatFold(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "VatFold Suite")
+}
+
+var _ = BeforeSuite(func() {
+	log.SetOutput(ioutil.Discard)
+})

test_config/test_config.go

@@ -75,6 +75,7 @@ func NewTestDB(node core.Node) *postgres.DB {
 func CleanTestDB(db *postgres.DB) {
 	db.MustExec("DELETE FROM blocks")
 	db.MustExec("DELETE FROM headers")
+	db.MustExec("DELETE FROM checked_headers")
 	db.MustExec("DELETE FROM log_filters")
 	db.MustExec("DELETE FROM logs")
 	db.MustExec("DELETE FROM maker.bite")
@@ -91,6 +92,7 @@ func CleanTestDB(db *postgres.DB) {
 	db.MustExec("DELETE FROM maker.tend")
 	db.MustExec("DELETE FROM maker.vat_grab")
 	db.MustExec("DELETE FROM maker.vat_init")
+	db.MustExec("DELETE FROM maker.vat_fold")
 	db.MustExec("DELETE FROM maker.vat_toll")
 	db.MustExec("DELETE FROM maker.vat_tune")
 	db.MustExec("DELETE FROM receipts")