Merge pull request #93 from vulcanize/vdb-570-storage-duplicates

(VDB-570) Handle duplicate storage diffs

.gitignore

@@ -14,3 +14,5 @@ postgraphile/package-lock.json
 vulcanizedb.log
 db/migrations/20*.sql
 plugins/*.so
+postgraphile/*.toml
+postgraphile/schema.graphql

Gopkg.lock

@@ -542,7 +542,6 @@
     "github.com/ethereum/go-ethereum/ethdb",
     "github.com/ethereum/go-ethereum/p2p",
     "github.com/ethereum/go-ethereum/p2p/discv5",
-    "github.com/ethereum/go-ethereum/params",
     "github.com/ethereum/go-ethereum/rlp",
     "github.com/ethereum/go-ethereum/rpc",
     "github.com/hashicorp/golang-lru",

Makefile

@@ -70,7 +70,8 @@ build: dep
 HOST_NAME = localhost
 PORT = 5432
 NAME =
-CONNECT_STRING=postgresql://$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable
+USER = postgres
+CONNECT_STRING=postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable
 
 # Parameter checks
 ## Check that DB variables are provided
@@ -79,6 +80,7 @@ checkdbvars:
 	test -n "$(HOST_NAME)" # $$HOST_NAME
 	test -n "$(PORT)" # $$PORT
 	test -n "$(NAME)" # $$NAME
+	test -n "$(USER)" # $$USER
 	@echo $(CONNECT_STRING)
 
 ## Check that the migration variable (id/timestamp) is provided

README.md

@@ -19,7 +19,8 @@
 
 ## Background
 The same data structures and encodings that make Ethereum an effective and trust-less distributed virtual machine
-complicate data accessibility and usability for dApp developers.
+complicate data accessibility and usability for dApp developers. VulcanizeDB improves Ethereum data accessibility by 
+providing a suite of tools to ease the extraction and transformation of data into a more useful state.
 
 
 ## Dependencies
@@ -60,6 +61,7 @@ It can be additionally helpful to add `$GOPATH/bin` to your shell's `$PATH`.
 1. `createdb vulcanize_public`
 1. `cd $GOPATH/src/github.com/vulcanize/vulcanizedb`
 1.  Run the migrations: `make migrate HOST_NAME=localhost NAME=vulcanize_public PORT=5432`
+    - There is an optional var `USER=username` if the database user is not the default user `postgres`
     - To rollback a single step: `make rollback NAME=vulcanize_public`
     - To rollback to a certain migration: `make rollback_to MIGRATION=n NAME=vulcanize_public`
     - To see status of migrations: `make migration_status NAME=vulcanize_public`
@@ -100,17 +102,21 @@ Usage is broken up into two processes:
 
 ### Data syncing
 To provide data for transformations, raw Ethereum data must first be synced into vDB.
-This is accomplished through the use of the `lightSync`, `sync`, or `coldImport` commands.
+This is accomplished through the use of the `headerSync`, `sync`, or `coldImport` commands.
 These commands are described in detail [here](../staging/documentation/sync.md).
 
 ### Data transformation
 Contract watchers use the raw data that has been synced into Postgres to filter out and apply transformations to specific data of interest.
 
-There is a built-in `contractWatcher` command which provides generic transformation of most contract data. This command is described in detail [here](../staging/documentation/contractWatcher.md).
+There is a built-in `contractWatcher` command which provides generic transformation of most contract data.
+The `contractWatcher` command is described further [here](../staging/documentation/contractWatcher.md).
 
 In many cases a custom transformer or set of transformers will need to be written to provide complete or more comprehensive coverage or to optimize other aspects of the output for a specific end-use.
-In this case we have provided the `compose`, `execute`, and `composeAndExecute` commands for running custom transformers from external repositories. This is described in detail [here](../staging/documentation/composeAndExecute.md).
+In this case we have provided the `compose`, `execute`, and `composeAndExecute` commands for running custom transformers from external repositories.
 
+Usage of the `compose`, `execute`, and `composeAndExecute` commands is described further [here](../staging/documentation/composeAndExecute.md).
+
+Documentation on how to build custom transformers to work with these commands can be found [here](../staging/documentation/transformers.md).
 
 ## Tests
 - Replace the empty `ipcPath` in the `environments/infura.toml` with a path to a full node's eth_jsonrpc endpoint (e.g. local geth node ipc path or infura url)

cmd/composeAndExecute.go

@@ -16,16 +16,20 @@
 package cmd
 
 import (
+	"os"
+	"plugin"
+	syn "sync"
+	"time"
+
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
 	"github.com/vulcanize/vulcanizedb/pkg/fs"
 	p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
 	"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
 	"github.com/vulcanize/vulcanizedb/utils"
-	"os"
-	"plugin"
-	syn "sync"
 )
 
 // composeAndExecuteCmd represents the composeAndExecute command
@@ -170,7 +174,8 @@ func composeAndExecute() {
 
 	if len(ethStorageInitializers) > 0 {
 		tailer := fs.FileTailer{Path: storageDiffsPath}
-		sw := watcher.NewStorageWatcher(tailer, &db)
+		storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
+		sw := watcher.NewStorageWatcher(storageFetcher, &db)
 		sw.AddTransformers(ethStorageInitializers)
 		wg.Add(1)
 		go watchEthStorage(&sw, &wg)
@@ -187,5 +192,6 @@ func composeAndExecute() {
 
 func init() {
 	rootCmd.AddCommand(composeAndExecuteCmd)
-	composeAndExecuteCmd.Flags().BoolVar(&recheckHeadersArg, "recheckHeaders", false, "checks headers that are already checked for each transformer.")
+	composeAndExecuteCmd.Flags().BoolVarP(&recheckHeadersArg, "recheck-headers", "r", false, "whether to re-check headers for watched events")
+	composeAndExecuteCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "how often to recheck queued storage diffs")
 }

cmd/contractWatcher.go

@@ -26,7 +26,7 @@ import (
 
 	st "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	ft "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer"
-	lt "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/transformer"
+	lt "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/transformer"
 	"github.com/vulcanize/vulcanizedb/utils"
 )
 
@@ -98,7 +98,7 @@ func contractWatcher() {
 	con := config.ContractConfig{}
 	con.PrepConfig()
 	switch mode {
-	case "light":
+	case "header":
 		t = lt.NewTransformer(con, blockChain, &db)
 	case "full":
 		t = ft.NewTransformer(con, blockChain, &db)
@@ -121,5 +121,5 @@ func contractWatcher() {
 
 func init() {
 	rootCmd.AddCommand(contractWatcherCmd)
-	contractWatcherCmd.Flags().StringVarP(&mode, "mode", "o", "light", "'light' or 'full' mode to work with either light synced or fully synced vDB (default is light)")
+	contractWatcherCmd.Flags().StringVarP(&mode, "mode", "o", "header", "'header' or 'full' mode to work with either header synced or fully synced vDB (default is header)")
 }

cmd/execute.go

@@ -26,6 +26,8 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	storageUtils "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
 	"github.com/vulcanize/vulcanizedb/pkg/fs"
@@ -118,7 +120,8 @@ func execute() {
 
 	if len(ethStorageInitializers) > 0 {
 		tailer := fs.FileTailer{Path: storageDiffsPath}
-		sw := watcher.NewStorageWatcher(tailer, &db)
+		storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
+		sw := watcher.NewStorageWatcher(storageFetcher, &db)
 		sw.AddTransformers(ethStorageInitializers)
 		wg.Add(1)
 		go watchEthStorage(&sw, &wg)
@@ -135,7 +138,8 @@ func execute() {
 
 func init() {
 	rootCmd.AddCommand(executeCmd)
-	executeCmd.Flags().BoolVar(&recheckHeadersArg, "recheckHeaders", false, "checks headers that are already checked for each transformer.")
+	executeCmd.Flags().BoolVarP(&recheckHeadersArg, "recheck-headers", "r", false, "whether to re-check headers for watched events")
+	executeCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "how often to recheck queued storage diffs")
 }
 
 type Exporter interface {
@@ -166,7 +170,9 @@ func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()
 	for range ticker.C {
-		w.Execute()
+		errs := make(chan error)
+		rows := make(chan storageUtils.StorageDiffRow)
+		w.Execute(rows, errs, queueRecheckInterval)
 	}
 }
 

cmd/headerSync.go

@@ -30,14 +30,14 @@ import (
 	"github.com/vulcanize/vulcanizedb/utils"
 )
 
-// lightSyncCmd represents the lightSync command
+// headerSyncCmd represents the headerSync command
-var lightSyncCmd = &cobra.Command{
+var headerSyncCmd = &cobra.Command{
-	Use:   "lightSync",
+	Use:   "headerSync",
 	Short: "Syncs VulcanizeDB with local ethereum node's block headers",
 	Long: `Syncs VulcanizeDB with local ethereum node. Populates
 Postgres with block headers.
 
-./vulcanizedb lightSync --starting-block-number 0 --config public.toml
+./vulcanizedb headerSync --starting-block-number 0 --config public.toml
 
 Expects ethereum node to be running and requires a .toml config:
 
@@ -50,13 +50,13 @@ Expects ethereum node to be running and requires a .toml config:
   ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
 `,
 	Run: func(cmd *cobra.Command, args []string) {
-		lightSync()
+		headerSync()
 	},
 }
 
 func init() {
-	rootCmd.AddCommand(lightSyncCmd)
+	rootCmd.AddCommand(headerSyncCmd)
-	lightSyncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
+	headerSyncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
 }
 
 func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
@@ -69,7 +69,7 @@ func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.H
 	missingBlocksPopulated <- populated
 }
 
-func lightSync() {
+func headerSync() {
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()
 	blockChain := getBlockChain()
@@ -86,7 +86,7 @@ func lightSync() {
 		case <-ticker.C:
 			window, err := validator.ValidateHeaders()
 			if err != nil {
-				log.Error("lightSync: ValidateHeaders failed: ", err)
+				log.Error("headerSync: ValidateHeaders failed: ", err)
 			}
 			log.Info(window.GetString())
 		case n := <-missingBlocksPopulated:

cmd/root.go

@@ -36,16 +36,17 @@ import (
 )
 
 var (
-	cfgFile             string
+	cfgFile              string
-	databaseConfig      config.Database
+	databaseConfig       config.Database
-	genConfig           config.Plugin
+	genConfig            config.Plugin
-	ipc                 string
+	ipc                  string
-	levelDbPath         string
+	levelDbPath          string
-	startingBlockNumber int64
+	queueRecheckInterval time.Duration
-	storageDiffsPath    string
+	startingBlockNumber  int64
-	syncAll             bool
+	storageDiffsPath     string
-	endingBlockNumber   int64
+	syncAll              bool
-	recheckHeadersArg   bool
+	endingBlockNumber    int64
+	recheckHeadersArg    bool
 )
 
 const (

db/migrations/00025_create_header_sync_transactions_table.sql

@@ -1,5 +1,5 @@
 -- +goose Up
-CREATE TABLE light_sync_transactions (
+CREATE TABLE header_sync_transactions (
   id          SERIAL PRIMARY KEY,
   header_id   INTEGER NOT NULL REFERENCES headers(id) ON DELETE CASCADE,
   hash        VARCHAR(66),
@@ -16,4 +16,4 @@ CREATE TABLE light_sync_transactions (
 );
 
 -- +goose Down
-DROP TABLE light_sync_transactions;
+DROP TABLE header_sync_transactions;

db/migrations/00026_create_header_sync_receipts_table.sql

@@ -1,7 +1,7 @@
 -- +goose Up
-CREATE TABLE light_sync_receipts(
+CREATE TABLE header_sync_receipts(
   id                  SERIAL PRIMARY KEY,
-  transaction_id      INTEGER NOT NULL REFERENCES light_sync_transactions(id) ON DELETE CASCADE,
+  transaction_id      INTEGER NOT NULL REFERENCES header_sync_transactions(id) ON DELETE CASCADE,
   header_id           INTEGER NOT NULL REFERENCES headers(id) ON DELETE CASCADE,
   contract_address    VARCHAR(42),
   cumulative_gas_used NUMERIC,
@@ -15,4 +15,4 @@ CREATE TABLE light_sync_receipts(
 
 
 -- +goose Down
-DROP TABLE light_sync_receipts;
+DROP TABLE header_sync_receipts;

db/schema.sql

@@ -259,6 +259,84 @@ CREATE SEQUENCE public.goose_db_version_id_seq
 ALTER SEQUENCE public.goose_db_version_id_seq OWNED BY public.goose_db_version.id;
 
 
+--
+-- Name: header_sync_receipts; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.header_sync_receipts (
+    id integer NOT NULL,
+    transaction_id integer NOT NULL,
+    header_id integer NOT NULL,
+    contract_address character varying(42),
+    cumulative_gas_used numeric,
+    gas_used numeric,
+    state_root character varying(66),
+    status integer,
+    tx_hash character varying(66),
+    rlp bytea
+);
+
+
+--
+-- Name: header_sync_receipts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.header_sync_receipts_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: header_sync_receipts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.header_sync_receipts_id_seq OWNED BY public.header_sync_receipts.id;
+
+
+--
+-- Name: header_sync_transactions; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.header_sync_transactions (
+    id integer NOT NULL,
+    header_id integer NOT NULL,
+    hash character varying(66),
+    gas_limit numeric,
+    gas_price numeric,
+    input_data bytea,
+    nonce numeric,
+    raw bytea,
+    tx_from character varying(44),
+    tx_index integer,
+    tx_to character varying(44),
+    value numeric
+);
+
+
+--
+-- Name: header_sync_transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.header_sync_transactions_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: header_sync_transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.header_sync_transactions_id_seq OWNED BY public.header_sync_transactions.id;
+
+
 --
 -- Name: headers; Type: TABLE; Schema: public; Owner: -
 --
@@ -294,84 +372,6 @@ CREATE SEQUENCE public.headers_id_seq
 ALTER SEQUENCE public.headers_id_seq OWNED BY public.headers.id;
 
 
---
--- Name: light_sync_receipts; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.light_sync_receipts (
-    id integer NOT NULL,
-    transaction_id integer NOT NULL,
-    header_id integer NOT NULL,
-    contract_address character varying(42),
-    cumulative_gas_used numeric,
-    gas_used numeric,
-    state_root character varying(66),
-    status integer,
-    tx_hash character varying(66),
-    rlp bytea
-);
-
-
---
--- Name: light_sync_receipts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.light_sync_receipts_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: light_sync_receipts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.light_sync_receipts_id_seq OWNED BY public.light_sync_receipts.id;
-
-
---
--- Name: light_sync_transactions; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.light_sync_transactions (
-    id integer NOT NULL,
-    header_id integer NOT NULL,
-    hash character varying(66),
-    gas_limit numeric,
-    gas_price numeric,
-    input_data bytea,
-    nonce numeric,
-    raw bytea,
-    tx_from character varying(44),
-    tx_index integer,
-    tx_to character varying(44),
-    value numeric
-);
-
-
---
--- Name: light_sync_transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.light_sync_transactions_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: light_sync_transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.light_sync_transactions_id_seq OWNED BY public.light_sync_transactions.id;
-
-
 --
 -- Name: log_filters; Type: TABLE; Schema: public; Owner: -
 --
@@ -619,6 +619,20 @@ ALTER TABLE ONLY public.full_sync_transactions ALTER COLUMN id SET DEFAULT nextv
 ALTER TABLE ONLY public.goose_db_version ALTER COLUMN id SET DEFAULT nextval('public.goose_db_version_id_seq'::regclass);
 
 
+--
+-- Name: header_sync_receipts id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_receipts ALTER COLUMN id SET DEFAULT nextval('public.header_sync_receipts_id_seq'::regclass);
+
+
+--
+-- Name: header_sync_transactions id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_transactions ALTER COLUMN id SET DEFAULT nextval('public.header_sync_transactions_id_seq'::regclass);
+
+
 --
 -- Name: headers id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -626,20 +640,6 @@ ALTER TABLE ONLY public.goose_db_version ALTER COLUMN id SET DEFAULT nextval('pu
 ALTER TABLE ONLY public.headers ALTER COLUMN id SET DEFAULT nextval('public.headers_id_seq'::regclass);
 
 
---
--- Name: light_sync_receipts id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.light_sync_receipts ALTER COLUMN id SET DEFAULT nextval('public.light_sync_receipts_id_seq'::regclass);
-
-
---
--- Name: light_sync_transactions id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.light_sync_transactions ALTER COLUMN id SET DEFAULT nextval('public.light_sync_transactions_id_seq'::regclass);
-
-
 --
 -- Name: log_filters id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -739,6 +739,38 @@ ALTER TABLE ONLY public.goose_db_version
     ADD CONSTRAINT goose_db_version_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: header_sync_receipts header_sync_receipts_header_id_transaction_id_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_receipts
+    ADD CONSTRAINT header_sync_receipts_header_id_transaction_id_key UNIQUE (header_id, transaction_id);
+
+
+--
+-- Name: header_sync_receipts header_sync_receipts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_receipts
+    ADD CONSTRAINT header_sync_receipts_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: header_sync_transactions header_sync_transactions_header_id_hash_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_transactions
+    ADD CONSTRAINT header_sync_transactions_header_id_hash_key UNIQUE (header_id, hash);
+
+
+--
+-- Name: header_sync_transactions header_sync_transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_transactions
+    ADD CONSTRAINT header_sync_transactions_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: headers headers_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -747,38 +779,6 @@ ALTER TABLE ONLY public.headers
     ADD CONSTRAINT headers_pkey PRIMARY KEY (id);
 
 
---
--- Name: light_sync_receipts light_sync_receipts_header_id_transaction_id_key; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.light_sync_receipts
-    ADD CONSTRAINT light_sync_receipts_header_id_transaction_id_key UNIQUE (header_id, transaction_id);
-
-
---
--- Name: light_sync_receipts light_sync_receipts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.light_sync_receipts
-    ADD CONSTRAINT light_sync_receipts_pkey PRIMARY KEY (id);
-
-
---
--- Name: light_sync_transactions light_sync_transactions_header_id_hash_key; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.light_sync_transactions
-    ADD CONSTRAINT light_sync_transactions_header_id_hash_key UNIQUE (header_id, hash);
-
-
---
--- Name: light_sync_transactions light_sync_transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.light_sync_transactions
-    ADD CONSTRAINT light_sync_transactions_pkey PRIMARY KEY (id);
-
-
 --
 -- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -909,6 +909,30 @@ ALTER TABLE ONLY public.full_sync_transactions
     ADD CONSTRAINT full_sync_transactions_block_id_fkey FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE;
 
 
+--
+-- Name: header_sync_receipts header_sync_receipts_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_receipts
+    ADD CONSTRAINT header_sync_receipts_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
+
+
+--
+-- Name: header_sync_receipts header_sync_receipts_transaction_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_receipts
+    ADD CONSTRAINT header_sync_receipts_transaction_id_fkey FOREIGN KEY (transaction_id) REFERENCES public.header_sync_transactions(id) ON DELETE CASCADE;
+
+
+--
+-- Name: header_sync_transactions header_sync_transactions_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_transactions
+    ADD CONSTRAINT header_sync_transactions_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
+
+
 --
 -- Name: headers headers_eth_node_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
@@ -917,30 +941,6 @@ ALTER TABLE ONLY public.headers
     ADD CONSTRAINT headers_eth_node_id_fkey FOREIGN KEY (eth_node_id) REFERENCES public.eth_nodes(id) ON DELETE CASCADE;
 
 
---
--- Name: light_sync_receipts light_sync_receipts_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.light_sync_receipts
-    ADD CONSTRAINT light_sync_receipts_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
-
-
---
--- Name: light_sync_receipts light_sync_receipts_transaction_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.light_sync_receipts
-    ADD CONSTRAINT light_sync_receipts_transaction_id_fkey FOREIGN KEY (transaction_id) REFERENCES public.light_sync_transactions(id) ON DELETE CASCADE;
-
-
---
--- Name: light_sync_transactions light_sync_transactions_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.light_sync_transactions
-    ADD CONSTRAINT light_sync_transactions_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
-
-
 --
 -- Name: blocks node_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
 --

dockerfiles/README.md

@@ -2,7 +2,7 @@ S
 `Dockerfile` will build an alpine image containing:
 - vDB as a binary with runtime deps statically linked: `/app/vulcanizedb`
 - The migration tool goose: `/app/goose`
-- Two services for running `lightSync` and `continuousLogSync`, started with the default configuration `environments/staging.toml`.
+- Two services for running `headerSync` and `continuousLogSync`, started with the default configuration `environments/staging.toml`.
 
 By default, vDB is configured towards the Kovan deploy. The configuration values can be overridden using environment variables, using the same hierarchical naming pattern but in CAPS and using underscores. For example, the contract address for the `Pit` can be set with the variable `CONTRACT_ADDRESS_PIT="0x123..."`.
 
@@ -18,8 +18,8 @@ e`
   * `CONTRACT_ADDRESS_[CONTRACT NAME]=0x123...`
   * `CONTRACT_ABI_[CONTRACT NAME]="ABI STRING"`
   * `CONTRACT_DEPLOYMENT-BLOCK_[CONTRACT NAME]=0` (doesn't really matter on a short chain, just avoids long unnecessary searching)
-6. Start the `lightSync` and `continuousLogSync` services:
+6. Start the `headerSync` and `continuousLogSync` services:
-  * `./vulcanizedb lightSync --config environments/staging.toml`
+  * `./vulcanizedb headerSync --config environments/staging.toml`
   * `./vulcanizedb continuousLogSync --config environments/staging.toml`
 
 ### Automated

dockerfiles/startup_script.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-# Runs the migrations and starts the lightSync and continuousLogSync services
+# Runs the migrations and starts the headerSync and continuousLogSync services
 
 # Exit if the variable tests fail
 set -e
@@ -21,7 +21,7 @@ set +e
 ./goose postgres "$CONNECT_STRING" up
 if [ $? -eq 0 ]; then
   # Fire up the services
-  ./vulcanizedb lightSync --config environments/staging.toml &
+  ./vulcanizedb headerSync --config environments/staging.toml &
   ./vulcanizedb continuousLogSync --config environments/staging.toml &
 else
   echo "Could not run migrations. Are the database details correct?"

documentation/composeAndExecute.md

@@ -3,48 +3,56 @@ The `composeAndExecute` command is used to compose and execute over an arbitrary
 This is accomplished by generating a Go pluggin which allows the `vulcanizedb` binary to link to external transformers, so
 long as they abide by one of the standard [interfaces](../staging/libraries/shared/transformer).
 
-This command requires Go 1.11+ and [Go plugins](https://golang.org/pkg/plugin/) only work on Unix-based systems.
+Additionally, there are separate `compose` and `execute` commands to allow pre-building and linking to a pre-built .so file.
 
-## Writing custom transformers
+**NOTE:**
-Storage Transformers
+1. It is necessary that the .so file was built with the same exact dependencies that are present in the execution environment,
-   * [Guide](../../staging/libraries/shared/factories/storage/README.md)
+i.e. we need to `compose` and `execute` the plugin .so file with the same exact version of vulcanizeDB.
-   * [Example](../../staging/libraries/shared/factories/storage/EXAMPLE.md)
+1. The plugin migrations are run during the plugin's composition. As such, if `execute` is used to run a prebuilt .so in a different
+environment than the one it was composed in then the migrations for that plugin will first need to be manually ran against that environment's Postgres database.
 
-Event Transformers   
+These commands require Go 1.11+ and use [Go plugins](https://golang.org/pkg/plugin/) which only work on Unix-based systems.
-   * [Guide](../../staging/libraries/shared/factories/event/README.md)
+There is also an ongoing [conflict](https://github.com/golang/go/issues/20481) between Go plugins and the use vendored dependencies which
-   * [Example 1](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registar)
+imposes certain limitations on how the plugins are built.
-   * [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registry)
-   * [Example 3](https://github.com/vulcanize/ens_transformers/tree/master/transformers/resolver)
 
-Contract Transformers
+## Commands
-   * [Example 1](https://github.com/vulcanize/account_transformers)
+The `compose` and `composeAndExecute` commands assume you are in the vulcanizdb directory located at your system's `$GOPATH`,
-   * [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/domain_records)
+and that all of the transformer repositories for building the plugin are present at their `$GOPATH` directories.
 
-## Preparing transformers to work as a plugin for composeAndExecute
+The `execute` command does not require the plugin transformer dependencies be located in their
-To plug in an external transformer we need to:
+`$GOPATH` directories, instead it expects a prebuilt .so file (of the name specified in the config file)
+to be in `$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins/` and, as noted above, also expects the plugin
+db migrations to have already been ran against the database.
 
-1. Create a package that exports a variable `TransformerInitializer`, `StorageTransformerInitializer`, or `ContractTransformerInitializer` that are of type [TransformerInitializer](../staging/libraries/shared/transformer/event_transformer.go#L33)
+compose:
-or [StorageTransformerInitializer](../../staging/libraries/shared/transformer/storage_transformer.go#L31),
-or [ContractTransformerInitializer](../../staging/libraries/shared/transformer/contract_transformer.go#L31), respectively
-2. Design the transformers to work in the context of their [event](../staging/libraries/shared/watcher/event_watcher.go#L83),
-[storage](../../staging/libraries/shared/watcher/storage_watcher.go#L53),
-or [contract](../../staging/libraries/shared/watcher/contract_watcher.go#L68) watcher execution modes
-3. Create db migrations to run against vulcanizeDB so that we can store the transformer output
-    * Do not `goose fix` the transformer migrations, this is to ensure they are always ran after the core vulcanizedb migrations which are kept in their fixed form
-    * Specify migration locations for each transformer in the config with the `exporter.transformer.migrations` fields
-    * If the base vDB migrations occupy this path as well, they need to be in their `goose fix`ed form
-    as they are [here](../../staging/db/migrations)
 
-To update a plugin repository with changes to the core vulcanizedb repository, replace the vulcanizedb vendored in the plugin repo (`plugin_repo/vendor/github.com/vulcanize/vulcanizedb`)
+`./vulcanizedb compose --config=./environments/config_name.toml`
-with the newly updated version
+
-* The entire vendor lib within the vendored vulcanizedb needs to be deleted (`plugin_repo/vendor/github.com/vulcanize/vulcanizedb/vendor`)
+execute:
-* These complications arise due to this [conflict](https://github.com/golang/go/issues/20481) between `dep` and Go plugins
+
+`./vulcanizedb execute --config=./environments/config_name.toml`
+
+composeAndExecute:
 
-## Configuration
-A .toml config file is specified when executing the command:
 `./vulcanizedb composeAndExecute --config=./environments/config_name.toml`
 
-The config provides information for composing a set of transformers:
+## Flags
+
+The `compose` and `composeAndExecute` commands can be passed optional flags to specify the operation of the watchers:
+
+- `--recheck-headers`/`-r` - specifies whether to re-check headers for events after the header has already been queried for watched logs.
+Can be useful for redundancy if you suspect that your node is not always returning all desired logs on every query.
+Argument is expected to be a boolean: e.g. `-r=true`.
+Defaults to `false`.
+
+- `query-recheck-interval`/`-q` - specifies interval for re-checking storage diffs that haven been queued for later processing
+(by default, the storage watched queues storage diffs if transformer execution fails, on the assumption that subsequent data derived from the event transformers may enable us to decode storage keys that we don't recognize right now).
+Argument is expected to be a duration (integer measured in nanoseconds): e.g. `-q=10m30s` (for 10 minute, 30 second intervals).
+Defaults to `5m` (5 minutes).
+
+## Configuration
+A .toml config file is specified when executing the commands.
+The config provides information for composing a set of transformers from external repositories:
 
 ```toml
 [database]
@@ -107,7 +115,7 @@ The config provides information for composing a set of transformers:
          that fetches event logs from an ETH node
         - `eth_contract` indicates the transformer works with the [contract watcher](../staging/libraries/shared/watcher/contract_watcher.go)
         that is made to work with [contract_watcher pkg](../../staging/pkg/contract_watcher)
-        based transformers which work with either a light or full sync vDB to watch events and poll public methods ([example1](https://github.com/vulcanize/account_transformers/tree/master/transformers/account/light), [example2](https://github.com/vulcanize/ens_transformers/tree/working/transformers/domain_records))
+        based transformers which work with either a header or full sync vDB to watch events and poll public methods ([example1](https://github.com/vulcanize/account_transformers/tree/master/transformers/account/light), [example2](https://github.com/vulcanize/ens_transformers/tree/working/transformers/domain_records))
     - `migrations` is the relative path from `repository` to the db migrations directory for the transformer
     - `rank` determines the order that migrations are ran, with lower ranked migrations running first
         - this is to help isolate any potential conflicts between transformer migrations

documentation/contractWatcher.md

@@ -5,11 +5,11 @@ It also provides some state variable coverage by automating polling of public me
 1. The method's arguments must all be of type address or bytes32 (hash)
 1. The method must return a single value
 
-This command operates in two modes- `light` and `full`- which require a light or full-synced vulcanizeDB, respectively.
+This command operates in two modes- `header` and `full`- which require a header or full-synced vulcanizeDB, respectively.
 
 This command requires the contract ABI be available on Etherscan if it is not provided in the config file by the user.
 
-If method polling is turned on we require an archival node at the ETH ipc endpoint in our config, whether or not we are operating in `light` or `full` mode.
+If method polling is turned on we require an archival node at the ETH ipc endpoint in our config, whether or not we are operating in `header` or `full` mode.
 Otherwise we only need to connect to a full node.
 
 ## Configuration
@@ -97,16 +97,16 @@ Modify `./environments/example.toml` to replace the empty `ipcPath` with a path
 This endpoint should be for an archival eth node if we want to perform method polling as this configuration is currently set up to do. To work with a non-archival full node,
 remove the `balanceOf` method from the `0x8dd5fbce2f6a956c3022ba3663759011dd51e73e` (TrueUSD) contract.
 
-If you are operating a light sync vDB, run:
+If you are operating a header sync vDB, run:
 
- `./vulcanizedb contractWatcher --config=./environments/example.toml --mode=light`
+ `./vulcanizedb contractWatcher --config=./environments/example.toml --mode=header`
 
 If instead you are operating a full sync vDB and provided an archival node IPC path, run in full mode:
 
  `./vulcanizedb contractWatcher --config=./environments/example.toml --mode=full`
 
 This will run the contractWatcher and configures it to watch the contracts specified in the config file. Note that
-by default we operate in `light` mode but the flag is included here to demonstrate its use.
+by default we operate in `header` mode but the flag is included here to demonstrate its use.
 
 The example config we link to in this example watches two contracts, the ENS Registry (0x314159265dD8dbb310642f98f50C066173C1259b) and TrueUSD (0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E).
 
@@ -117,43 +117,43 @@ The TrueUSD contract is configured with two events (`Transfer` and `Mint`) and a
 to poll the `balanceOf` method with those addresses at every block. Note that we do not provide an ABI for TrueUSD as its ABI can be fetched from Etherscan.
 
 For the ENS contract, it produces and populates a schema with four tables"
-`light_0x314159265dd8dbb310642f98f50c066173c1259b.newowner_event`
+`header_0x314159265dd8dbb310642f98f50c066173c1259b.newowner_event`
-`light_0x314159265dd8dbb310642f98f50c066173c1259b.newresolver_event`
+`header_0x314159265dd8dbb310642f98f50c066173c1259b.newresolver_event`
-`light_0x314159265dd8dbb310642f98f50c066173c1259b.newttl_event`
+`header_0x314159265dd8dbb310642f98f50c066173c1259b.newttl_event`
-`light_0x314159265dd8dbb310642f98f50c066173c1259b.transfer_event`
+`header_0x314159265dd8dbb310642f98f50c066173c1259b.transfer_event`
 
 For the TrusUSD contract, it produces and populates a schema with three tables:
 
-`light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event`
+`header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event`
-`light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.mint_event`
+`header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.mint_event`
-`light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method`
+`header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method`
 
 Column ids and types for these tables are generated based on the event and method argument names and types and method return types, resulting in tables such as:
 
-Table "light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event"
+Table "header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event"
 
-|  Column    |         Type          | Collation | Nullable |                                           Default                                           | Storage  | Stats target | Description  
+|  Column    |         Type          | Collation | Nullable |                                           Default                                            | Storage  | Stats target | Description  
-|:----------:|:---------------------:|:---------:|:--------:|:-------------------------------------------------------------------------------------------:|:--------:|:------------:|:-----------:|
+|:----------:|:---------------------:|:---------:|:--------:|:--------------------------------------------------------------------------------------------:|:--------:|:------------:|:-----------:|
-| id         | integer               |           | not null | nextval('light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event_id_seq'::regclass) | plain    |              |             |
+| id         | integer               |           | not null | nextval('header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event_id_seq'::regclass) | plain    |              |             |
-| header_id  | integer               |           | not null |                                                                                             | plain    |              |             |
+| header_id  | integer               |           | not null |                                                                                              | plain    |              |             |
-| token_name | character varying(66) |           | not null |                                                                                             | extended |              |             |
+| token_name | character varying(66) |           | not null |                                                                                              | extended |              |             |
-| raw_log    | jsonb                 |           |          |                                                                                             | extended |              |             |
+| raw_log    | jsonb                 |           |          |                                                                                              | extended |              |             |
-| log_idx    | integer               |           | not null |                                                                                             | plain    |              |             |
+| log_idx    | integer               |           | not null |                                                                                              | plain    |              |             |
-| tx_idx     | integer               |           | not null |                                                                                             | plain    |              |             |
+| tx_idx     | integer               |           | not null |                                                                                              | plain    |              |             |
-| from_      | character varying(66) |           | not null |                                                                                             | extended |              |             |
+| from_      | character varying(66) |           | not null |                                                                                              | extended |              |             |
-| to_        | character varying(66) |           | not null |                                                                                             | extended |              |             |
+| to_        | character varying(66) |           | not null |                                                                                              | extended |              |             |
-| value_     | numeric               |           | not null |                                                                                             | main     |              |             |
+| value_     | numeric               |           | not null |                                                                                              | main     |              |             |
 
 
-Table "light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method"
+Table "header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method"
 
-|   Column   |         Type          | Collation | Nullable |                                            Default                                            | Storage  | Stats target | Description |
+|   Column   |         Type          | Collation | Nullable |                                            Default                                             | Storage  | Stats target | Description |
-|:----------:|:---------------------:|:---------:|:--------:|:-------------------------------------------------------------------------------------------:|:--------:|:------------:|:-----------:|
+|:----------:|:---------------------:|:---------:|:--------:|:----------------------------------------------------------------------------------------------:|:--------:|:------------:|:-----------:|
-| id         | integer               |           | not null | nextval('light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method_id_seq'::regclass) | plain    |              |             |
+| id         | integer               |           | not null | nextval('header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method_id_seq'::regclass) | plain    |              |             |
-| token_name | character varying(66) |           | not null |                                                                                               | extended |              |             |
+| token_name | character varying(66) |           | not null |                                                                                                | extended |              |             |
-| block      | integer               |           | not null |                                                                                               | plain    |              |             |
+| block      | integer               |           | not null |                                                                                                | plain    |              |             |
-| who_       | character varying(66) |           | not null |                                                                                               | extended |              |             |
+| who_       | character varying(66) |           | not null |                                                                                                | extended |              |             |
-| returned   | numeric               |           | not null |                                                                                               | main     |              |             |
+| returned   | numeric               |           | not null |                                                                                                | main     |              |             |
   
 The addition of '_' after table names is to prevent collisions with reserved Postgres words.
 

documentation/sync.md

@@ -1,12 +1,12 @@
 # Syncing commands
 These commands are used to sync raw Ethereum data into Postgres.
 
-## lightSync
+## headerSync
 Syncs VulcanizeDB with the configured Ethereum node, populating only block headers.
 This command is useful when you want a minimal baseline from which to track targeted data on the blockchain (e.g. individual smart contract storage values or event logs).
 1. Start Ethereum node
 1. In a separate terminal start VulcanizeDB:
-    - `./vulcanizedb lightSync --config <config.toml> --starting-block-number <block-number>`
+    - `./vulcanizedb headerSync --config <config.toml> --starting-block-number <block-number>`
     
 ## sync
 Syncs VulcanizeDB with the configured Ethereum node, populating blocks, transactions, receipts, and logs.

documentation/transformers.md

@@ -0,0 +1,40 @@
+# Custom transformers
+When the capabilities of the generic `contractWatcher` are not sufficient, custom transformers tailored to a specific
+purpose can be leveraged.
+
+Individual transformers can be composed together from any number of external repositories and executed as a single process using
+the `compose` and `execute` commands or the `composeAndExecute` command.
+
+## Writing custom transformers
+For help with writing different types of custom transformers for the `composeAndExecute` set of commands, please see the below:
+
+Storage Transformers
+   * [Guide](../../staging/libraries/shared/factories/storage/README.md)
+   * [Example](../../staging/libraries/shared/factories/storage/EXAMPLE.md)
+
+Event Transformers
+   * [Guide](../../staging/libraries/shared/factories/event/README.md)
+   * [Example 1](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registar)
+   * [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registry)
+   * [Example 3](https://github.com/vulcanize/ens_transformers/tree/master/transformers/resolver)
+
+Contract Transformers
+   * [Example 1](https://github.com/vulcanize/account_transformers)
+   * [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/domain_records)
+
+## Preparing custom transformers to work as part of a plugin
+To plug in an external transformer we need to:
+
+1. Create a package that exports a variable `TransformerInitializer`, `StorageTransformerInitializer`, or `ContractTransformerInitializer` that are of type [TransformerInitializer](../staging/libraries/shared/transformer/event_transformer.go#L33)
+or [StorageTransformerInitializer](../../staging/libraries/shared/transformer/storage_transformer.go#L31),
+or [ContractTransformerInitializer](../../staging/libraries/shared/transformer/contract_transformer.go#L31), respectively
+2. Design the transformers to work in the context of their [event](../staging/libraries/shared/watcher/event_watcher.go#L83),
+[storage](../../staging/libraries/shared/watcher/storage_watcher.go#L53),
+or [contract](../../staging/libraries/shared/watcher/contract_watcher.go#L68) watcher execution modes
+3. Create db migrations to run against vulcanizeDB so that we can store the transformer output
+    * Do not `goose fix` the transformer migrations, this is to ensure they are always ran after the core vulcanizedb migrations which are kept in their fixed form
+    * Specify migration locations for each transformer in the config with the `exporter.transformer.migrations` fields
+    * If the base vDB migrations occupy this path as well, they need to be in their `goose fix`ed form
+    as they are [here](../../staging/db/migrations)
+
+To update a plugin repository with changes to the core vulcanizedb repository, run `dep ensure` to update its dependencies.

integration_test/contract_watcher_full_transformer_test.go

@@ -286,7 +286,7 @@ var _ = Describe("contractWatcher full transformer", func() {
 			err = t.Execute()
 			Expect(err).ToNot(HaveOccurred())
 
-			log := test_helpers.LightNewOwnerLog{}
+			log := test_helpers.HeaderSyncNewOwnerLog{}
 			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("does not exist"))

integration_test/contract_watcher_header_sync_transformer_test.go

@@ -9,7 +9,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks"
@@ -18,7 +18,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 )
 
-var _ = Describe("contractWatcher light transformer", func() {
+var _ = Describe("contractWatcher headerSync transformer", func() {
 	var db *postgres.DB
 	var err error
 	var blockChain core.BlockChain
@@ -49,6 +49,13 @@ var _ = Describe("contractWatcher light transformer", func() {
 			c, ok := t.Contracts[tusdAddr]
 			Expect(ok).To(Equal(true))
 
+			// TODO: Fix this
+			// This test sometimes randomly fails because
+			// for some reason the starting block number is not updated from
+			// its original value (5197514) to the block number (6194632)
+			// of the earliest header (mocks.MockHeader1) in the repository
+			// It is not clear how this happens without one of the above insertErrs
+			// having been thrown and without any errors thrown during the Init() call
 			Expect(c.StartingBlock).To(Equal(int64(6194632)))
 			Expect(c.Abi).To(Equal(constants.TusdAbiString))
 			Expect(c.Name).To(Equal("TrueUSD"))
@@ -100,8 +107,8 @@ var _ = Describe("contractWatcher light transformer", func() {
 			err = t.Execute()
 			Expect(err).ToNot(HaveOccurred())
 
-			log := test_helpers.LightTransferLog{}
+			log := test_helpers.HeaderSyncTransferLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&log)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event", tusdAddr)).StructScan(&log)
 			Expect(err).ToNot(HaveOccurred())
 			// We don't know vulcID, so compare individual fields instead of complete structures
 			Expect(log.HeaderID).To(Equal(headerID))
@@ -168,12 +175,12 @@ var _ = Describe("contractWatcher light transformer", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			res := test_helpers.BalanceOf{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res.Balance).To(Equal("55849938025000000000000"))
 			Expect(res.TokenName).To(Equal("TrueUSD"))
 
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("no rows in result set"))
 		})
@@ -208,8 +215,8 @@ var _ = Describe("contractWatcher light transformer", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(t.Start).To(Equal(int64(6885698)))
 
-			log := test_helpers.LightNewOwnerLog{}
+			log := test_helpers.HeaderSyncNewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&log)
 			Expect(err).ToNot(HaveOccurred())
 			// We don't know vulcID, so compare individual fields instead of complete structures
 			Expect(log.HeaderID).To(Equal(headerID))
@@ -260,17 +267,17 @@ var _ = Describe("contractWatcher light transformer", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			res := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
 			Expect(res.TokenName).To(Equal(""))
 
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
 			Expect(res.TokenName).To(Equal(""))
 
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("no rows in result set"))
 		})
@@ -287,8 +294,8 @@ var _ = Describe("contractWatcher light transformer", func() {
 			err = t.Execute()
 			Expect(err).ToNot(HaveOccurred())
 
-			log := test_helpers.LightNewOwnerLog{}
+			log := test_helpers.HeaderSyncNewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&log)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("does not exist"))
 		})
@@ -309,12 +316,12 @@ var _ = Describe("contractWatcher light transformer", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			res := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
 			Expect(res.TokenName).To(Equal(""))
 
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("no rows in result set"))
 		})
@@ -338,16 +345,16 @@ var _ = Describe("contractWatcher light transformer", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(t.Start).To(Equal(int64(6885702)))
 
-			newOwnerLog := test_helpers.LightNewOwnerLog{}
+			newOwnerLog := test_helpers.HeaderSyncNewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
 			Expect(err).ToNot(HaveOccurred())
 			// We don't know vulcID, so compare individual fields instead of complete structures
 			Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
 			Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
 			Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
 
-			transferLog := test_helpers.LightTransferLog{}
+			transferLog := test_helpers.HeaderSyncTransferLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
 			Expect(err).ToNot(HaveOccurred())
 			// We don't know vulcID, so compare individual fields instead of complete structures
 			Expect(transferLog.From).To(Equal("0x8cA465764873E71CEa525F5EB6AE973d650c22C2"))
@@ -410,27 +417,27 @@ var _ = Describe("contractWatcher light transformer", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			owner := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(owner.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
 			Expect(owner.TokenName).To(Equal(""))
 
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(owner.Address).To(Equal("0x0000000000000000000000000000000000000000"))
 			Expect(owner.TokenName).To(Equal(""))
 
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("no rows in result set"))
 
 			bal := test_helpers.BalanceOf{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x8cA465764873E71CEa525F5EB6AE973d650c22C2' AND block = '6885701'", tusdAddr)).StructScan(&bal)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x8cA465764873E71CEa525F5EB6AE973d650c22C2' AND block = '6885701'", tusdAddr)).StructScan(&bal)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(bal.Balance).To(Equal("1954436000000000000000"))
 			Expect(bal.TokenName).To(Equal("TrueUSD"))
 
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6885701'", tusdAddr)).StructScan(&bal)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6885701'", tusdAddr)).StructScan(&bal)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("no rows in result set"))
 		})

integration_test/poller_test.go

@@ -130,10 +130,10 @@ var _ = Describe("Poller", func() {
 		})
 	})
 
-	Describe("Light sync mode", func() {
+	Describe("Header sync mode", func() {
 		BeforeEach(func() {
 			db, bc = test_helpers.SetupDBandBC()
-			contractPoller = poller.NewPoller(bc, db, types.LightSync)
+			contractPoller = poller.NewPoller(bc, db, types.HeaderSync)
 		})
 
 		Describe("PollContract", func() {
@@ -148,22 +148,22 @@ var _ = Describe("Poller", func() {
 
 				scanStruct := test_helpers.BalanceOf{}
 
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(scanStruct.Balance).To(Equal("66386309548896882859581786"))
 				Expect(scanStruct.TokenName).To(Equal("TrueUSD"))
 
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707323'", constants.TusdContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707323'", constants.TusdContractAddress)).StructScan(&scanStruct)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(scanStruct.Balance).To(Equal("66386309548896882859581786"))
 				Expect(scanStruct.TokenName).To(Equal("TrueUSD"))
 
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(scanStruct.Balance).To(Equal("17982350181394112023885864"))
 				Expect(scanStruct.TokenName).To(Equal("TrueUSD"))
 
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE' AND block = '6707323'", constants.TusdContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE' AND block = '6707323'", constants.TusdContractAddress)).StructScan(&scanStruct)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(scanStruct.Balance).To(Equal("17982350181394112023885864"))
 				Expect(scanStruct.TokenName).To(Equal("TrueUSD"))
@@ -180,12 +180,12 @@ var _ = Describe("Poller", func() {
 
 				scanStruct := test_helpers.Owner{}
 
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x7e74a86b6e146964fb965db04dc2590516da77f720bb6759337bf5632415fd86' AND block = '6885877'", constants.EnsContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x7e74a86b6e146964fb965db04dc2590516da77f720bb6759337bf5632415fd86' AND block = '6885877'", constants.EnsContractAddress)).StructScan(&scanStruct)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(scanStruct.Address).To(Equal("0x546aA2EaE2514494EeaDb7bbb35243348983C59d"))
 				Expect(scanStruct.TokenName).To(Equal("ENS-Registry"))
 
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885877'", constants.EnsContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885877'", constants.EnsContractAddress)).StructScan(&scanStruct)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(scanStruct.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
 				Expect(scanStruct.TokenName).To(Equal("ENS-Registry"))
@@ -202,7 +202,7 @@ var _ = Describe("Poller", func() {
 
 				scanStruct := test_helpers.BalanceOf{}
 
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
 				Expect(err).To(HaveOccurred())
 			})
 
@@ -217,7 +217,7 @@ var _ = Describe("Poller", func() {
 				Expect(err).ToNot(HaveOccurred())
 
 				scanStruct := test_helpers.Resolver{}
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.resolver_method WHERE node_ = '0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8' AND block = '6921967'", constants.EnsContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.resolver_method WHERE node_ = '0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8' AND block = '6921967'", constants.EnsContractAddress)).StructScan(&scanStruct)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(scanStruct.Address).To(Equal("0x5FfC014343cd971B7eb70732021E26C35B744cc4"))
 				Expect(scanStruct.TokenName).To(Equal("ENS-Registry"))
@@ -225,13 +225,13 @@ var _ = Describe("Poller", func() {
 
 				test_helpers.TearDown(db)
 				db, bc = test_helpers.SetupDBandBC()
-				contractPoller = poller.NewPoller(bc, db, types.LightSync)
+				contractPoller = poller.NewPoller(bc, db, types.HeaderSync)
 
 				con.Piping = true
 				err = contractPoller.PollContract(*con, 6921968)
 				Expect(err).ToNot(HaveOccurred())
 
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.resolver_method WHERE node_ = '0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8' AND block = '6921967'", constants.EnsContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.resolver_method WHERE node_ = '0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8' AND block = '6921967'", constants.EnsContractAddress)).StructScan(&scanStruct)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(scanStruct.Address).To(Equal("0x5FfC014343cd971B7eb70732021E26C35B744cc4"))
 				Expect(scanStruct.TokenName).To(Equal("ENS-Registry"))

libraries/shared/factories/event/README.md

@@ -2,13 +2,13 @@
 	
 One approach VulcanizeDB takes to caching and indexing smart contracts is to watch contract events emitted in receipt logs.
 
-With a light synced vDB we can watch events by iterating over headers retrieved from the synced `headers` table and using these headers to
+With a header synced vDB we can watch events by iterating over headers retrieved from the synced `headers` table and using these headers to
 fetch and verify relevant event logs from a full Ethereum node, keeping track of which headers we have checked for which events 
 with our `checked_headers` table.
 
 ## Assumptions
 
-This approach assumes you are running a vDB light sync which is itself run against a light Ethereum node,
+This approach assumes you are running a vDB header sync which is run against a light Ethereum node;
 this approach also assumes there is a full node available. 
 
 Looking forward, we will be building fetchers that enable sourcing data from IPFS instead of an ETH node.
@@ -17,7 +17,7 @@ Looking forward, we will be building fetchers that enable sourcing data from IPF
 
 VulcanizeDB has shared code built out for building and plugging in event transformers
 
-### [Event Watcher (light sync)](../staging/libraries/shared/watcher/event_watcher.go)
+### [Event Watcher (header sync)](../staging/libraries/shared/watcher/event_watcher.go)
 
 The event watcher is responsible for continuously fetching and delegating chunks of logs and their associated header to the appropriate transformers.
 

libraries/shared/factories/event/log_note_transformer.go

@@ -1,73 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package event
-
-import (
-	"github.com/ethereum/go-ethereum/core/types"
-	log "github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-)
-
-type LogNoteTransformer struct {
-	Config     transformer.EventTransformerConfig
-	Converter  LogNoteConverter
-	Repository Repository
-}
-
-func (tr LogNoteTransformer) NewLogNoteTransformer(db *postgres.DB) transformer.EventTransformer {
-	tr.Repository.SetDB(db)
-	return tr
-}
-
-func (tr LogNoteTransformer) Execute(logs []types.Log, header core.Header, recheckedHeader constants.TransformerExecution) error {
-	transformerName := tr.Config.TransformerName
-
-	// No matching logs, mark the header as checked for this type of logs
-	if len(logs) < 1 {
-		err := tr.Repository.MarkHeaderChecked(header.Id)
-		if err != nil {
-			log.Printf("Error marking header as checked in %v: %v", transformerName, err)
-			return err
-		}
-		return nil
-	}
-
-	models, err := tr.Converter.ToModels(logs)
-	if err != nil {
-		log.Printf("Error converting logs in %v: %v", transformerName, err)
-		return err
-	}
-
-	err = tr.Repository.Create(header.Id, models)
-	if err != nil {
-		log.Printf("Error persisting %v record: %v", transformerName, err)
-		return err
-	}
-	return nil
-}
-
-func (tr LogNoteTransformer) GetName() string {
-	return tr.Config.TransformerName
-}
-
-func (tr LogNoteTransformer) GetConfig() transformer.EventTransformerConfig {
-	return tr.Config
-}

libraries/shared/factories/event/log_note_transformer_test.go

@@ -1,126 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package event_test
-
-import (
-	"math/rand"
-
-	"github.com/ethereum/go-ethereum/core/types"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/event"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/fakes"
-)
-
-var _ = Describe("LogNoteTransformer", func() {
-	var (
-		repository mocks.MockRepository
-		converter  mocks.MockLogNoteConverter
-		headerOne  core.Header
-		t          transformer.EventTransformer
-		model      test_data.GenericModel
-		config     = test_data.GenericTestConfig
-		logs       = test_data.GenericTestLogs
-	)
-
-	BeforeEach(func() {
-		repository = mocks.MockRepository{}
-		converter = mocks.MockLogNoteConverter{}
-		t = event.LogNoteTransformer{
-			Config:     config,
-			Converter:  &converter,
-			Repository: &repository,
-		}.NewLogNoteTransformer(nil)
-
-		headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()}
-	})
-
-	It("sets the database", func() {
-		Expect(repository.SetDbCalled).To(BeTrue())
-	})
-
-	It("marks header checked if no logs are provided", func() {
-		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
-
-		Expect(err).NotTo(HaveOccurred())
-		repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id)
-	})
-
-	It("doesn't attempt to convert or persist an empty collection when there are no logs", func() {
-		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.ToModelsCalledCounter).To(Equal(0))
-		Expect(repository.CreateCalledCounter).To(Equal(0))
-	})
-
-	It("does not call repository.MarkCheckedHeader when there are logs", func() {
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
-
-		Expect(err).NotTo(HaveOccurred())
-		repository.AssertMarkHeaderCheckedNotCalled()
-	})
-
-	It("returns error if marking header checked returns err", func() {
-		repository.SetMarkHeaderCheckedError(fakes.FakeError)
-
-		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
-
-		Expect(err).To(HaveOccurred())
-		Expect(err).To(MatchError(fakes.FakeError))
-	})
-
-	It("converts matching logs to models", func() {
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.PassedLogs).To(Equal(logs))
-	})
-
-	It("returns error if converter returns error", func() {
-		converter.SetConverterError(fakes.FakeError)
-
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
-
-		Expect(err).To(HaveOccurred())
-		Expect(err).To(MatchError(fakes.FakeError))
-	})
-
-	It("persists the model", func() {
-		converter.SetReturnModels([]interface{}{model})
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(repository.PassedHeaderID).To(Equal(headerOne.Id))
-		Expect(repository.PassedModels).To(Equal([]interface{}{model}))
-	})
-
-	It("returns error if repository returns error for create", func() {
-		repository.SetCreateError(fakes.FakeError)
-
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
-
-		Expect(err).To(HaveOccurred())
-		Expect(err).To(MatchError(fakes.FakeError))
-	})
-})

libraries/shared/fetcher/log_fetcher.go

@@ -24,22 +24,22 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
-type LogFetcher interface {
+type ILogFetcher interface {
 	FetchLogs(contractAddresses []common.Address, topics []common.Hash, missingHeader core.Header) ([]types.Log, error)
 }
 
-type Fetcher struct {
+type LogFetcher struct {
 	blockChain core.BlockChain
 }
 
-func NewFetcher(blockchain core.BlockChain) *Fetcher {
+func NewLogFetcher(blockchain core.BlockChain) *LogFetcher {
-	return &Fetcher{
+	return &LogFetcher{
 		blockChain: blockchain,
 	}
 }
 
 // Checks all topic0s, on all addresses, fetching matching logs for the given header
-func (fetcher Fetcher) FetchLogs(addresses []common.Address, topic0s []common.Hash, header core.Header) ([]types.Log, error) {
+func (logFetcher LogFetcher) FetchLogs(addresses []common.Address, topic0s []common.Hash, header core.Header) ([]types.Log, error) {
 	blockHash := common.HexToHash(header.Hash)
 	query := ethereum.FilterQuery{
 		BlockHash: &blockHash,
@@ -48,7 +48,7 @@ func (fetcher Fetcher) FetchLogs(addresses []common.Address, topic0s []common.Ha
 		Topics: [][]common.Hash{topic0s},
 	}
 
-	logs, err := fetcher.blockChain.GetEthLogsWithCustomQuery(query)
+	logs, err := logFetcher.blockChain.GetEthLogsWithCustomQuery(query)
 	if err != nil {
 		// TODO review aggregate fetching error handling
 		return []types.Log{}, err

libraries/shared/fetcher/log_fetcher_test.go

@@ -22,16 +22,16 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	fetch "github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 )
 
-var _ = Describe("Fetcher", func() {
+var _ = Describe("LogFetcher", func() {
 	Describe("FetchLogs", func() {
 		It("fetches logs based on the given query", func() {
 			blockChain := fakes.NewMockBlockChain()
-			fetcher := fetch.NewFetcher(blockChain)
+			logFetcher := fetcher.NewLogFetcher(blockChain)
 			header := fakes.FakeHeader
 
 			addresses := []common.Address{
@@ -41,7 +41,7 @@ var _ = Describe("Fetcher", func() {
 
 			topicZeros := []common.Hash{common.BytesToHash([]byte{1, 2, 3, 4, 5})}
 
-			_, err := fetcher.FetchLogs(addresses, topicZeros, header)
+			_, err := logFetcher.FetchLogs(addresses, topicZeros, header)
 
 			address1 := common.HexToAddress("0xfakeAddress")
 			address2 := common.HexToAddress("0xanotherFakeAddress")
@@ -59,9 +59,9 @@ var _ = Describe("Fetcher", func() {
 		It("returns an error if fetching the logs fails", func() {
 			blockChain := fakes.NewMockBlockChain()
 			blockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError)
-			fetcher := fetch.NewFetcher(blockChain)
+			logFetcher := fetcher.NewLogFetcher(blockChain)
 
-			_, err := fetcher.FetchLogs([]common.Address{}, []common.Hash{}, core.Header{})
+			_, err := logFetcher.FetchLogs([]common.Address{}, []common.Hash{}, core.Header{})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err).To(MatchError(fakes.FakeError))

libraries/shared/fetcher/storage_fetcher.go

@@ -0,0 +1,50 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package fetcher
+
+import (
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/fs"
+	"strings"
+)
+
+type IStorageFetcher interface {
+	FetchStorageDiffs(chan<- utils.StorageDiffRow, chan<- error)
+}
+
+type CsvTailStorageFetcher struct {
+	tailer fs.Tailer
+}
+
+func NewCsvTailStorageFetcher(tailer fs.Tailer) CsvTailStorageFetcher {
+	return CsvTailStorageFetcher{tailer: tailer}
+}
+
+func (storageFetcher CsvTailStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffRow, errs chan<- error) {
+	t, tailErr := storageFetcher.tailer.Tail()
+	if tailErr != nil {
+		errs <- tailErr
+	}
+	for line := range t.Lines {
+		row, parseErr := utils.FromStrings(strings.Split(line.Text, ","))
+		if parseErr != nil {
+			errs <- parseErr
+		} else {
+			out <- row
+		}
+	}
+}

libraries/shared/fetcher/storage_fetcher_test.go

@@ -0,0 +1,99 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package fetcher_test
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/hpcloud/tail"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+)
+
+var _ = Describe("Csv Tail Storage Fetcher", func() {
+	var (
+		errorsChannel  chan error
+		mockTailer     *fakes.MockTailer
+		rowsChannel    chan utils.StorageDiffRow
+		storageFetcher fetcher.CsvTailStorageFetcher
+	)
+
+	BeforeEach(func() {
+		errorsChannel = make(chan error)
+		rowsChannel = make(chan utils.StorageDiffRow)
+		mockTailer = fakes.NewMockTailer()
+		storageFetcher = fetcher.NewCsvTailStorageFetcher(mockTailer)
+	})
+
+	It("adds error to errors channel if tailing file fails", func(done Done) {
+		mockTailer.TailErr = fakes.FakeError
+
+		go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel)
+
+		Expect(<-errorsChannel).To(MatchError(fakes.FakeError))
+		close(done)
+	})
+
+	It("adds parsed csv row to rows channel for storage diff", func(done Done) {
+		line := getFakeLine()
+
+		go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel)
+		mockTailer.Lines <- line
+
+		expectedRow, err := utils.FromStrings(strings.Split(line.Text, ","))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(<-rowsChannel).To(Equal(expectedRow))
+		close(done)
+	})
+
+	It("adds error to errors channel if parsing csv fails", func(done Done) {
+		line := &tail.Line{Text: "invalid"}
+
+		go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel)
+		mockTailer.Lines <- line
+
+		Expect(<-errorsChannel).To(HaveOccurred())
+		select {
+		case <-rowsChannel:
+			Fail("value passed to rows channel on error")
+		default:
+			Succeed()
+		}
+		close(done)
+	})
+})
+
+func getFakeLine() *tail.Line {
+	address := common.HexToAddress("0x1234567890abcdef")
+	blockHash := []byte{4, 5, 6}
+	blockHeight := int64(789)
+	storageKey := []byte{9, 8, 7}
+	storageValue := []byte{6, 5, 4}
+	return &tail.Line{
+		Text: fmt.Sprintf("%s,%s,%d,%s,%s", common.Bytes2Hex(address.Bytes()), common.Bytes2Hex(blockHash),
+			blockHeight, common.Bytes2Hex(storageKey), common.Bytes2Hex(storageValue)),
+		Time: time.Time{},
+		Err:  nil,
+	}
+}

libraries/shared/mocks/converter.go

@@ -53,6 +53,6 @@ func (converter *MockConverter) SetToEntityConverterError(err error) {
 	converter.entityConverterError = err
 }
 
-func (c *MockConverter) SetToModelConverterError(err error) {
+func (converter *MockConverter) SetToModelConverterError(err error) {
-	c.modelConverterError = err
+	converter.modelConverterError = err
 }

libraries/shared/mocks/storage_fetcher.go

@@ -14,10 +14,26 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package event
+package mocks
 
-import "github.com/ethereum/go-ethereum/core/types"
+import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 
-type LogNoteConverter interface {
+type MockStorageFetcher struct {
-	ToModels(ethLog []types.Log) ([]interface{}, error)
+	RowsToReturn []utils.StorageDiffRow
+	ErrsToReturn []error
+}
+
+func NewMockStorageFetcher() *MockStorageFetcher {
+	return &MockStorageFetcher{}
+}
+
+func (fetcher *MockStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffRow, errs chan<- error) {
+	defer close(out)
+	defer close(errs)
+	for _, err := range fetcher.ErrsToReturn {
+		errs <- err
+	}
+	for _, row := range fetcher.RowsToReturn {
+		out <- row
+	}
 }

libraries/shared/mocks/storage_queue.go

@@ -21,11 +21,26 @@ import (
 )
 
 type MockStorageQueue struct {
-	AddCalled bool
+	AddCalled      bool
-	AddError  error
+	AddError       error
+	AddPassedRow   utils.StorageDiffRow
+	DeleteErr      error
+	DeletePassedId int
+	GetAllErr      error
+	RowsToReturn   []utils.StorageDiffRow
 }
 
 func (queue *MockStorageQueue) Add(row utils.StorageDiffRow) error {
 	queue.AddCalled = true
+	queue.AddPassedRow = row
 	return queue.AddError
 }
+
+func (queue *MockStorageQueue) Delete(id int) error {
+	queue.DeletePassedId = id
+	return queue.DeleteErr
+}
+
+func (queue *MockStorageQueue) GetAll() ([]utils.StorageDiffRow, error) {
+	return queue.RowsToReturn, queue.GetAllErr
+}

libraries/shared/repository/repository_test.go

@@ -26,7 +26,7 @@ import (
 
 	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
 	shared "github.com/vulcanize/vulcanizedb/libraries/shared/repository"
-	r2 "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository"
+	r2 "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"

libraries/shared/storage/storage_queue.go

@@ -23,6 +23,8 @@ import (
 
 type IStorageQueue interface {
 	Add(row utils.StorageDiffRow) error
+	Delete(id int) error
+	GetAll() ([]utils.StorageDiffRow, error)
 }
 
 type StorageQueue struct {
@@ -40,3 +42,14 @@ func (queue StorageQueue) Add(row utils.StorageDiffRow) error {
 		row.BlockHeight, row.StorageKey.Bytes(), row.StorageValue.Bytes())
 	return err
 }
+
+func (queue StorageQueue) Delete(id int) error {
+	_, err := queue.db.Exec(`DELETE FROM public.queued_storage WHERE id = $1`, id)
+	return err
+}
+
+func (queue StorageQueue) GetAll() ([]utils.StorageDiffRow, error) {
+	var result []utils.StorageDiffRow
+	err := queue.db.Select(&result, `SELECT * FROM public.queued_storage`)
+	return result, err
+}

libraries/shared/storage/storage_queue_test.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package storage_test
 
 import (
@@ -7,27 +23,79 @@ import (
 
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
 var _ = Describe("Storage queue", func() {
-	It("adds a storage row to the db", func() {
+	var (
-		row := utils.StorageDiffRow{
+		db    *postgres.DB
+		row   utils.StorageDiffRow
+		queue storage.IStorageQueue
+	)
+
+	BeforeEach(func() {
+		row = utils.StorageDiffRow{
 			Contract:     common.HexToAddress("0x123456"),
 			BlockHash:    common.HexToHash("0x678901"),
 			BlockHeight:  987,
 			StorageKey:   common.HexToHash("0x654321"),
 			StorageValue: common.HexToHash("0x198765"),
 		}
-		db := test_config.NewTestDB(test_config.NewTestNode())
+		db = test_config.NewTestDB(test_config.NewTestNode())
-		queue := storage.NewStorageQueue(db)
+		test_config.CleanTestDB(db)
-
+		queue = storage.NewStorageQueue(db)
 		addErr := queue.Add(row)
-
 		Expect(addErr).NotTo(HaveOccurred())
+	})
+
+	It("adds a storage row to the db", func() {
 		var result utils.StorageDiffRow
 		getErr := db.Get(&result, `SELECT contract, block_hash, block_height, storage_key, storage_value FROM public.queued_storage`)
 		Expect(getErr).NotTo(HaveOccurred())
 		Expect(result).To(Equal(row))
 	})
+
+	It("deletes storage row from db", func() {
+		rows, getErr := queue.GetAll()
+		Expect(getErr).NotTo(HaveOccurred())
+		Expect(len(rows)).To(Equal(1))
+
+		err := queue.Delete(rows[0].Id)
+
+		Expect(err).NotTo(HaveOccurred())
+		remainingRows, secondGetErr := queue.GetAll()
+		Expect(secondGetErr).NotTo(HaveOccurred())
+		Expect(len(remainingRows)).To(BeZero())
+	})
+
+	It("gets all storage rows from db", func() {
+		rowTwo := utils.StorageDiffRow{
+			Contract:     common.HexToAddress("0x123456"),
+			BlockHash:    common.HexToHash("0x678902"),
+			BlockHeight:  988,
+			StorageKey:   common.HexToHash("0x654322"),
+			StorageValue: common.HexToHash("0x198766"),
+		}
+		addErr := queue.Add(rowTwo)
+		Expect(addErr).NotTo(HaveOccurred())
+
+		rows, err := queue.GetAll()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(rows)).To(Equal(2))
+		Expect(rows[0]).NotTo(Equal(rows[1]))
+		Expect(rows[0].Id).NotTo(BeZero())
+		Expect(rows[0].Contract).To(Or(Equal(row.Contract), Equal(rowTwo.Contract)))
+		Expect(rows[0].BlockHash).To(Or(Equal(row.BlockHash), Equal(rowTwo.BlockHash)))
+		Expect(rows[0].BlockHeight).To(Or(Equal(row.BlockHeight), Equal(rowTwo.BlockHeight)))
+		Expect(rows[0].StorageKey).To(Or(Equal(row.StorageKey), Equal(rowTwo.StorageKey)))
+		Expect(rows[0].StorageValue).To(Or(Equal(row.StorageValue), Equal(rowTwo.StorageValue)))
+		Expect(rows[1].Id).NotTo(BeZero())
+		Expect(rows[1].Contract).To(Or(Equal(row.Contract), Equal(rowTwo.Contract)))
+		Expect(rows[1].BlockHash).To(Or(Equal(row.BlockHash), Equal(rowTwo.BlockHash)))
+		Expect(rows[1].BlockHeight).To(Or(Equal(row.BlockHeight), Equal(rowTwo.BlockHeight)))
+		Expect(rows[1].StorageKey).To(Or(Equal(row.StorageKey), Equal(rowTwo.StorageKey)))
+		Expect(rows[1].StorageValue).To(Or(Equal(row.StorageValue), Equal(rowTwo.StorageValue)))
+	})
 })

libraries/shared/storage/utils/errors.go

@@ -17,9 +17,12 @@
 package utils
 
 import (
+	"errors"
 	"fmt"
 )
 
+var ErrRowExists = errors.New("parsed row for storage diff already exists")
+
 type ErrContractNotFound struct {
 	Contract string
 }

libraries/shared/storage/utils/row.go

@@ -25,6 +25,7 @@ import (
 const ExpectedRowLength = 5
 
 type StorageDiffRow struct {
+	Id           int
 	Contract     common.Address
 	BlockHash    common.Hash `db:"block_hash"`
 	BlockHeight  int         `db:"block_height"`

libraries/shared/transactions/syncer.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package transactions
 
 import (
@@ -28,6 +44,9 @@ func NewTransactionsSyncer(db *postgres.DB, blockChain core.BlockChain) Transact
 
 func (syncer TransactionsSyncer) SyncTransactions(headerID int64, logs []types.Log) error {
 	transactionHashes := getUniqueTransactionHashes(logs)
+	if len(transactionHashes) < 1 {
+		return nil
+	}
 	transactions, transactionErr := syncer.BlockChain.GetTransactions(transactionHashes)
 	if transactionErr != nil {
 		return transactionErr

libraries/shared/transactions/syncer_test.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package transactions_test
 
 import (
@@ -24,12 +40,19 @@ var _ = Describe("Transaction syncer", func() {
 	})
 
 	It("fetches transactions for logs", func() {
-		err := syncer.SyncTransactions(0, []types.Log{})
+		err := syncer.SyncTransactions(0, []types.Log{{TxHash: fakes.FakeHash}})
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(blockChain.GetTransactionsCalled).To(BeTrue())
 	})
 
+	It("does not fetch transactions if no logs", func() {
+		err := syncer.SyncTransactions(0, []types.Log{})
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(blockChain.GetTransactionsCalled).To(BeFalse())
+	})
+
 	It("only fetches transactions with unique hashes", func() {
 		err := syncer.SyncTransactions(0, []types.Log{{
 			TxHash: fakes.FakeHash,
@@ -44,7 +67,7 @@ var _ = Describe("Transaction syncer", func() {
 	It("returns error if fetching transactions fails", func() {
 		blockChain.GetTransactionsError = fakes.FakeError
 
-		err := syncer.SyncTransactions(0, []types.Log{})
+		err := syncer.SyncTransactions(0, []types.Log{{TxHash: fakes.FakeHash}})
 
 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
@@ -55,7 +78,7 @@ var _ = Describe("Transaction syncer", func() {
 		mockHeaderRepository := fakes.NewMockHeaderRepository()
 		syncer.Repository = mockHeaderRepository
 
-		err := syncer.SyncTransactions(0, []types.Log{})
+		err := syncer.SyncTransactions(0, []types.Log{{TxHash: fakes.FakeHash}})
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(mockHeaderRepository.CreateTransactionsCalled).To(BeTrue())
@@ -67,7 +90,7 @@ var _ = Describe("Transaction syncer", func() {
 		mockHeaderRepository.CreateTransactionsError = fakes.FakeError
 		syncer.Repository = mockHeaderRepository
 
-		err := syncer.SyncTransactions(0, []types.Log{})
+		err := syncer.SyncTransactions(0, []types.Log{{TxHash: fakes.FakeHash}})
 
 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))

libraries/shared/transactions/transactions_suite_test.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package transactions_test
 
 import (

libraries/shared/watcher/contract_watcher.go

@@ -54,10 +54,10 @@ func (watcher *ContractWatcher) AddTransformers(inits interface{}) error {
 		watcher.Transformers = append(watcher.Transformers, t)
 	}
 
-	for _, transformer := range watcher.Transformers {
+	for _, contractTransformer := range watcher.Transformers {
-		err := transformer.Init()
+		err := contractTransformer.Init()
 		if err != nil {
-			log.Print("Unable to initialize transformer:", transformer.GetConfig().Name, err)
+			log.Print("Unable to initialize transformer:", contractTransformer.GetConfig().Name, err)
 			return err
 		}
 	}
@@ -65,10 +65,10 @@ func (watcher *ContractWatcher) AddTransformers(inits interface{}) error {
 }
 
 func (watcher *ContractWatcher) Execute() error {
-	for _, transformer := range watcher.Transformers {
+	for _, contractTransformer := range watcher.Transformers {
-		err := transformer.Execute()
+		err := contractTransformer.Execute()
 		if err != nil {
-			log.Error("Unable to execute transformer:", transformer.GetConfig().Name, err)
+			log.Error("Unable to execute transformer:", contractTransformer.GetConfig().Name, err)
 			return err
 		}
 	}

libraries/shared/watcher/event_watcher.go

@@ -37,7 +37,7 @@ type EventWatcher struct {
 	Transformers  []transformer.EventTransformer
 	BlockChain    core.BlockChain
 	DB            *postgres.DB
-	Fetcher       fetcher.LogFetcher
+	Fetcher       fetcher.ILogFetcher
 	Chunker       chunker.Chunker
 	Addresses     []common.Address
 	Topics        []common.Hash
@@ -47,7 +47,7 @@ type EventWatcher struct {
 
 func NewEventWatcher(db *postgres.DB, bc core.BlockChain) EventWatcher {
 	logChunker := chunker.NewLogChunker()
-	logFetcher := fetcher.NewFetcher(bc)
+	logFetcher := fetcher.NewLogFetcher(bc)
 	transactionSyncer := transactions.NewTransactionsSyncer(db, bc)
 	return EventWatcher{
 		BlockChain: bc,

libraries/shared/watcher/storage_watcher.go

@@ -17,73 +17,100 @@
 package watcher
 
 import (
+	"fmt"
 	"reflect"
-	"strings"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/sirupsen/logrus"
 
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/fs"
 )
 
 type StorageWatcher struct {
-	db           *postgres.DB
+	db             *postgres.DB
-	tailer       fs.Tailer
+	StorageFetcher fetcher.IStorageFetcher
-	Queue        storage.IStorageQueue
+	Queue          storage.IStorageQueue
-	Transformers map[common.Address]transformer.StorageTransformer
+	Transformers   map[common.Address]transformer.StorageTransformer
 }
 
-func NewStorageWatcher(tailer fs.Tailer, db *postgres.DB) StorageWatcher {
+func NewStorageWatcher(fetcher fetcher.IStorageFetcher, db *postgres.DB) StorageWatcher {
 	transformers := make(map[common.Address]transformer.StorageTransformer)
 	queue := storage.NewStorageQueue(db)
 	return StorageWatcher{
-		db:           db,
+		db:             db,
-		tailer:       tailer,
+		StorageFetcher: fetcher,
-		Queue:        queue,
+		Queue:          queue,
-		Transformers: transformers,
+		Transformers:   transformers,
 	}
 }
 
-func (watcher StorageWatcher) AddTransformers(initializers []transformer.StorageTransformerInitializer) {
+func (storageWatcher StorageWatcher) AddTransformers(initializers []transformer.StorageTransformerInitializer) {
 	for _, initializer := range initializers {
-		storageTransformer := initializer(watcher.db)
+		storageTransformer := initializer(storageWatcher.db)
-		watcher.Transformers[storageTransformer.ContractAddress()] = storageTransformer
+		storageWatcher.Transformers[storageTransformer.ContractAddress()] = storageTransformer
 	}
 }
 
-func (watcher StorageWatcher) Execute() error {
+func (storageWatcher StorageWatcher) Execute(rows chan utils.StorageDiffRow, errs chan error, queueRecheckInterval time.Duration) {
-	t, tailErr := watcher.tailer.Tail()
+	ticker := time.NewTicker(queueRecheckInterval)
-	if tailErr != nil {
+	go storageWatcher.StorageFetcher.FetchStorageDiffs(rows, errs)
-		return tailErr
+	for {
-	}
+		select {
-	for line := range t.Lines {
+		case fetchErr := <-errs:
-		row, parseErr := utils.FromStrings(strings.Split(line.Text, ","))
+			logrus.Warn(fmt.Sprintf("error fetching storage diffs: %s", fetchErr))
-		if parseErr != nil {
+		case row := <-rows:
-			return parseErr
+			storageWatcher.processRow(row)
+		case <-ticker.C:
+			storageWatcher.processQueue()
 		}
-		storageTransformer, ok := watcher.Transformers[row.Contract]
+	}
+}
+
+func (storageWatcher StorageWatcher) processRow(row utils.StorageDiffRow) {
+	storageTransformer, ok := storageWatcher.Transformers[row.Contract]
+	if !ok {
+		// ignore rows from unwatched contracts
+		return
+	}
+	executeErr := storageTransformer.Execute(row)
+	if executeErr != nil && executeErr != utils.ErrRowExists {
+		logrus.Warn(fmt.Sprintf("error executing storage transformer: %s", executeErr))
+		queueErr := storageWatcher.Queue.Add(row)
+		if queueErr != nil {
+			logrus.Warn(fmt.Sprintf("error queueing storage diff: %s", queueErr))
+		}
+	}
+}
+
+func (storageWatcher StorageWatcher) processQueue() {
+	rows, fetchErr := storageWatcher.Queue.GetAll()
+	if fetchErr != nil {
+		logrus.Warn(fmt.Sprintf("error getting queued storage: %s", fetchErr))
+	}
+	for _, row := range rows {
+		storageTransformer, ok := storageWatcher.Transformers[row.Contract]
 		if !ok {
-			logrus.Warn(utils.ErrContractNotFound{Contract: row.Contract.Hex()}.Error())
+			// delete row from queue if address no longer watched
+			storageWatcher.deleteRow(row.Id)
 			continue
 		}
 		executeErr := storageTransformer.Execute(row)
-		if executeErr != nil {
+		if executeErr == nil || executeErr == utils.ErrRowExists {
-			if isKeyNotFound(executeErr) {
+			storageWatcher.deleteRow(row.Id)
-				queueErr := watcher.Queue.Add(row)
-				if queueErr != nil {
-					logrus.Warn(queueErr.Error())
-				}
-			} else {
-				logrus.Warn(executeErr.Error())
-			}
-			continue
 		}
 	}
-	return nil
+}
+
+func (storageWatcher StorageWatcher) deleteRow(id int) {
+	deleteErr := storageWatcher.Queue.Delete(id)
+	if deleteErr != nil {
+		logrus.Warn(fmt.Sprintf("error deleting persisted row from queue: %s", deleteErr))
+	}
 }
 
 func isKeyNotFound(executeErr error) bool {

libraries/shared/watcher/storage_watcher_test.go

@@ -17,15 +17,11 @@
 package watcher_test
 
 import (
-	"errors"
-	"fmt"
 	"io/ioutil"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/hpcloud/tail"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/sirupsen/logrus"
@@ -34,7 +30,6 @@ import (
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
@@ -43,170 +38,230 @@ var _ = Describe("Storage Watcher", func() {
 	It("adds transformers", func() {
 		fakeAddress := common.HexToAddress("0x12345")
 		fakeTransformer := &mocks.MockStorageTransformer{Address: fakeAddress}
-		w := watcher.NewStorageWatcher(&fakes.MockTailer{}, test_config.NewTestDB(core.Node{}))
+		w := watcher.NewStorageWatcher(mocks.NewMockStorageFetcher(), test_config.NewTestDB(test_config.NewTestNode()))
 
 		w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
 
 		Expect(w.Transformers[fakeAddress]).To(Equal(fakeTransformer))
 	})
 
-	It("reads the tail of the storage diffs file", func() {
+	Describe("executing watcher", func() {
-		mockTailer := fakes.NewMockTailer()
+		var (
-		w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
+			errs            chan error
+			mockFetcher     *mocks.MockStorageFetcher
+			mockQueue       *mocks.MockStorageQueue
+			mockTransformer *mocks.MockStorageTransformer
+			row             utils.StorageDiffRow
+			rows            chan utils.StorageDiffRow
+			storageWatcher  watcher.StorageWatcher
+		)
 
-		assert(func(err error) {
+		BeforeEach(func() {
-			Expect(err).To(BeNil())
+			errs = make(chan error)
-			Expect(mockTailer.TailCalled).To(BeTrue())
+			rows = make(chan utils.StorageDiffRow)
-		}, w, mockTailer, []*tail.Line{})
+			address := common.HexToAddress("0x0123456789abcdef")
-	})
+			mockFetcher = mocks.NewMockStorageFetcher()
-
+			mockQueue = &mocks.MockStorageQueue{}
-	It("returns error if row parsing fails", func() {
+			mockTransformer = &mocks.MockStorageTransformer{Address: address}
-		mockTailer := fakes.NewMockTailer()
+			row = utils.StorageDiffRow{
-		w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
+				Id:           1337,
-		line := &tail.Line{Text: "oops"}
+				Contract:     address,
-
+				BlockHash:    common.HexToHash("0xfedcba9876543210"),
-		assert(func(err error) {
+				BlockHeight:  0,
-			Expect(err).To(HaveOccurred())
+				StorageKey:   common.HexToHash("0xabcdef1234567890"),
-			Expect(err).To(MatchError(utils.ErrRowMalformed{Length: 1}))
+				StorageValue: common.HexToHash("0x9876543210abcdef"),
-		}, w, mockTailer, []*tail.Line{line})
+			}
-	})
-
-	It("logs error if no transformer can parse storage row", func() {
-		mockTailer := fakes.NewMockTailer()
-		address := common.HexToAddress("0x12345")
-		line := getFakeLine(address.Bytes())
-		w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
-		tempFile, err := ioutil.TempFile("", "log")
-		defer os.Remove(tempFile.Name())
-		Expect(err).NotTo(HaveOccurred())
-		logrus.SetOutput(tempFile)
-
-		assert(func(err error) {
-			Expect(err).NotTo(HaveOccurred())
-			logContent, readErr := ioutil.ReadFile(tempFile.Name())
-			Expect(readErr).NotTo(HaveOccurred())
-			Expect(string(logContent)).To(ContainSubstring(utils.ErrContractNotFound{Contract: address.Hex()}.Error()))
-		}, w, mockTailer, []*tail.Line{line})
-	})
-
-	It("executes transformer with storage row", func() {
-		address := []byte{1, 2, 3}
-		line := getFakeLine(address)
-		mockTailer := fakes.NewMockTailer()
-		w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
-		fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address)}
-		w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
-
-		assert(func(err error) {
-			Expect(err).To(BeNil())
-			expectedRow, err := utils.FromStrings(strings.Split(line.Text, ","))
-			Expect(err).NotTo(HaveOccurred())
-			Expect(fakeTransformer.PassedRow).To(Equal(expectedRow))
-		}, w, mockTailer, []*tail.Line{line})
-	})
-
-	Describe("when executing transformer fails", func() {
-		It("queues row when error is storage key not found", func() {
-			address := []byte{1, 2, 3}
-			line := getFakeLine(address)
-			mockTailer := fakes.NewMockTailer()
-			w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
-			mockQueue := &mocks.MockStorageQueue{}
-			w.Queue = mockQueue
-			keyNotFoundError := utils.ErrStorageKeyNotFound{Key: "unknown_storage_key"}
-			fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address), ExecuteErr: keyNotFoundError}
-			w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
-
-			assert(func(err error) {
-				Expect(err).NotTo(HaveOccurred())
-				Expect(mockQueue.AddCalled).To(BeTrue())
-			}, w, mockTailer, []*tail.Line{line})
 		})
 
-		It("logs error if queuing row fails", func() {
+		It("logs error if fetching storage diffs fails", func(done Done) {
-			address := []byte{1, 2, 3}
+			mockFetcher.ErrsToReturn = []error{fakes.FakeError}
-			line := getFakeLine(address)
+			storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode()))
-			mockTailer := fakes.NewMockTailer()
+			storageWatcher.Queue = mockQueue
-			w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
+			storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer})
-			mockQueue := &mocks.MockStorageQueue{}
+			tempFile, fileErr := ioutil.TempFile("", "log")
-			mockQueue.AddError = fakes.FakeError
+			Expect(fileErr).NotTo(HaveOccurred())
-			w.Queue = mockQueue
-			keyNotFoundError := utils.ErrStorageKeyNotFound{Key: "unknown_storage_key"}
-			fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address), ExecuteErr: keyNotFoundError}
-			w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
-			tempFile, err := ioutil.TempFile("", "log")
 			defer os.Remove(tempFile.Name())
-			Expect(err).NotTo(HaveOccurred())
 			logrus.SetOutput(tempFile)
 
-			assert(func(err error) {
+			go storageWatcher.Execute(rows, errs, time.Hour)
-				Expect(err).NotTo(HaveOccurred())
+
-				Expect(mockQueue.AddCalled).To(BeTrue())
+			Eventually(func() (string, error) {
-				logContent, readErr := ioutil.ReadFile(tempFile.Name())
+				logContent, err := ioutil.ReadFile(tempFile.Name())
-				Expect(readErr).NotTo(HaveOccurred())
+				return string(logContent), err
-				Expect(string(logContent)).To(ContainSubstring(fakes.FakeError.Error()))
+			}).Should(ContainSubstring(fakes.FakeError.Error()))
-			}, w, mockTailer, []*tail.Line{line})
+			close(done)
 		})
 
-		It("logs any other error", func() {
+		Describe("transforming new storage diffs", func() {
-			address := []byte{1, 2, 3}
+			BeforeEach(func() {
-			line := getFakeLine(address)
+				mockFetcher.RowsToReturn = []utils.StorageDiffRow{row}
-			mockTailer := fakes.NewMockTailer()
+				storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode()))
-			w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
+				storageWatcher.Queue = mockQueue
-			executionError := errors.New("storage watcher failed attempting to execute transformer")
+				storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer})
-			fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address), ExecuteErr: executionError}
+			})
-			w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
-			tempFile, err := ioutil.TempFile("", "log")
-			defer os.Remove(tempFile.Name())
-			Expect(err).NotTo(HaveOccurred())
-			logrus.SetOutput(tempFile)
 
-			assert(func(err error) {
+			It("executes transformer for recognized storage row", func(done Done) {
-				Expect(err).NotTo(HaveOccurred())
+				go storageWatcher.Execute(rows, errs, time.Hour)
-				logContent, readErr := ioutil.ReadFile(tempFile.Name())
+
-				Expect(readErr).NotTo(HaveOccurred())
+				Eventually(func() utils.StorageDiffRow {
-				Expect(string(logContent)).To(ContainSubstring(executionError.Error()))
+					return mockTransformer.PassedRow
-			}, w, mockTailer, []*tail.Line{line})
+				}).Should(Equal(row))
+				close(done)
+			})
+
+			It("does not queue row if transformer execution fails because row already exists", func(done Done) {
+				mockTransformer.ExecuteErr = utils.ErrRowExists
+
+				go storageWatcher.Execute(rows, errs, time.Hour)
+
+				Expect(<-errs).To(BeNil())
+				Consistently(func() bool {
+					return mockQueue.AddCalled
+				}).Should(BeFalse())
+				close(done)
+			})
+
+			It("queues row for later processing if transformer execution fails", func(done Done) {
+				mockTransformer.ExecuteErr = fakes.FakeError
+
+				go storageWatcher.Execute(rows, errs, time.Hour)
+
+				Expect(<-errs).To(BeNil())
+				Eventually(func() bool {
+					return mockQueue.AddCalled
+				}).Should(BeTrue())
+				Eventually(func() utils.StorageDiffRow {
+					return mockQueue.AddPassedRow
+				}).Should(Equal(row))
+				close(done)
+			})
+
+			It("logs error if queueing row fails", func(done Done) {
+				mockTransformer.ExecuteErr = utils.ErrStorageKeyNotFound{}
+				mockQueue.AddError = fakes.FakeError
+				tempFile, fileErr := ioutil.TempFile("", "log")
+				Expect(fileErr).NotTo(HaveOccurred())
+				defer os.Remove(tempFile.Name())
+				logrus.SetOutput(tempFile)
+
+				go storageWatcher.Execute(rows, errs, time.Hour)
+
+				Eventually(func() bool {
+					return mockQueue.AddCalled
+				}).Should(BeTrue())
+				Eventually(func() (string, error) {
+					logContent, err := ioutil.ReadFile(tempFile.Name())
+					return string(logContent), err
+				}).Should(ContainSubstring(fakes.FakeError.Error()))
+				close(done)
+			})
 		})
+
+		Describe("transforming queued storage diffs", func() {
+			BeforeEach(func() {
+				mockQueue.RowsToReturn = []utils.StorageDiffRow{row}
+				storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode()))
+				storageWatcher.Queue = mockQueue
+				storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer})
+			})
+
+			It("logs error if getting queued storage fails", func(done Done) {
+				mockQueue.GetAllErr = fakes.FakeError
+				tempFile, fileErr := ioutil.TempFile("", "log")
+				Expect(fileErr).NotTo(HaveOccurred())
+				defer os.Remove(tempFile.Name())
+				logrus.SetOutput(tempFile)
+
+				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+
+				Eventually(func() (string, error) {
+					logContent, err := ioutil.ReadFile(tempFile.Name())
+					return string(logContent), err
+				}).Should(ContainSubstring(fakes.FakeError.Error()))
+				close(done)
+			})
+
+			It("executes transformer for storage row", func(done Done) {
+				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+
+				Eventually(func() utils.StorageDiffRow {
+					return mockTransformer.PassedRow
+				}).Should(Equal(row))
+				close(done)
+			})
+
+			It("deletes row from queue if transformer execution successful", func(done Done) {
+				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+
+				Eventually(func() int {
+					return mockQueue.DeletePassedId
+				}).Should(Equal(row.Id))
+				close(done)
+			})
+
+			It("deletes row from queue if transformer execution errors because row already exists", func(done Done) {
+				mockTransformer.ExecuteErr = utils.ErrRowExists
+
+				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+
+				Eventually(func() int {
+					return mockQueue.DeletePassedId
+				}).Should(Equal(row.Id))
+				close(done)
+			})
+
+			It("logs error if deleting persisted row fails", func(done Done) {
+				mockQueue.DeleteErr = fakes.FakeError
+				tempFile, fileErr := ioutil.TempFile("", "log")
+				Expect(fileErr).NotTo(HaveOccurred())
+				defer os.Remove(tempFile.Name())
+				logrus.SetOutput(tempFile)
+
+				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+
+				Eventually(func() (string, error) {
+					logContent, err := ioutil.ReadFile(tempFile.Name())
+					return string(logContent), err
+				}).Should(ContainSubstring(fakes.FakeError.Error()))
+				close(done)
+			})
+
+			It("deletes obsolete row from queue if contract not recognized", func(done Done) {
+				obsoleteRow := utils.StorageDiffRow{
+					Id:       row.Id + 1,
+					Contract: common.HexToAddress("0xfedcba9876543210"),
+				}
+				mockQueue.RowsToReturn = []utils.StorageDiffRow{obsoleteRow}
+
+				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+
+				Eventually(func() int {
+					return mockQueue.DeletePassedId
+				}).Should(Equal(obsoleteRow.Id))
+				close(done)
+			})
+
+			It("logs error if deleting obsolete row fails", func(done Done) {
+				obsoleteRow := utils.StorageDiffRow{
+					Id:       row.Id + 1,
+					Contract: common.HexToAddress("0xfedcba9876543210"),
+				}
+				mockQueue.RowsToReturn = []utils.StorageDiffRow{obsoleteRow}
+				mockQueue.DeleteErr = fakes.FakeError
+				tempFile, fileErr := ioutil.TempFile("", "log")
+				Expect(fileErr).NotTo(HaveOccurred())
+				defer os.Remove(tempFile.Name())
+				logrus.SetOutput(tempFile)
+
+				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+
+				Eventually(func() (string, error) {
+					logContent, err := ioutil.ReadFile(tempFile.Name())
+					return string(logContent), err
+				}).Should(ContainSubstring(fakes.FakeError.Error()))
+				close(done)
+			})
+		})
+
 	})
 })
-
-func assert(assertion func(err error), watcher watcher.StorageWatcher, mockTailer *fakes.MockTailer, lines []*tail.Line) {
-	errs := make(chan error, 1)
-	done := make(chan bool, 1)
-	go execute(watcher, errs, done)
-	for _, line := range lines {
-		mockTailer.Lines <- line
-	}
-	close(mockTailer.Lines)
-
-	select {
-	case err := <-errs:
-		assertion(err)
-		break
-	case <-done:
-		assertion(nil)
-		break
-	}
-}
-
-func execute(w watcher.StorageWatcher, errs chan error, done chan bool) {
-	err := w.Execute()
-	if err != nil {
-		errs <- err
-	} else {
-		done <- true
-	}
-}
-
-func getFakeLine(address []byte) *tail.Line {
-	blockHash := []byte{4, 5, 6}
-	blockHeight := int64(789)
-	storageKey := []byte{9, 8, 7}
-	storageValue := []byte{6, 5, 4}
-	return &tail.Line{
-		Text: fmt.Sprintf("%s,%s,%d,%s,%s", common.Bytes2Hex(address), common.Bytes2Hex(blockHash), blockHeight, common.Bytes2Hex(storageKey), common.Bytes2Hex(storageValue)),
-		Time: time.Time{},
-		Err:  nil,
-	}
-}

pkg/contract_watcher/full/transformer/transformer_test.go

@@ -39,7 +39,7 @@ var _ = Describe("Transformer", func() {
 
 	Describe("Init", func() {
 		It("Initializes transformer's contract objects", func() {
-			blockRetriever := &fakes.MockFullBlockRetriever{}
+			blockRetriever := &fakes.MockFullSyncBlockRetriever{}
 			firstBlock := int64(1)
 			mostRecentBlock := int64(2)
 			blockRetriever.FirstBlock = firstBlock
@@ -74,7 +74,7 @@ var _ = Describe("Transformer", func() {
 		})
 
 		It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() {
-			blockRetriever := &fakes.MockFullBlockRetriever{}
+			blockRetriever := &fakes.MockFullSyncBlockRetriever{}
 			blockRetriever.FirstBlockErr = fakes.FakeError
 			t := getTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
 

pkg/contract_watcher/header/converter/converter_suite_test.go

@@ -27,7 +27,7 @@ import (
 
 func TestConverter(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Light Converter Suite Test")
+	RunSpecs(t, "Header Sync Converter Suite Test")
 }
 
 var _ = BeforeSuite(func() {

pkg/contract_watcher/header/converter/converter_test.go

@@ -22,7 +22,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/converter"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/converter"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"

pkg/contract_watcher/header/fetcher/fetcher_test.go

@@ -22,7 +22,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/fetcher"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/fetcher"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 )

pkg/contract_watcher/header/repository/header_repository_test.go

@@ -23,7 +23,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -32,7 +32,7 @@ import (
 
 var _ = Describe("Repository", func() {
 	var db *postgres.DB
-	var contractHeaderRepo repository.HeaderRepository // contract_watcher light header repository
+	var contractHeaderRepo repository.HeaderRepository // contract_watcher headerSync header repository
 	var coreHeaderRepo repositories.HeaderRepository   // pkg/datastore header repository
 	var eventIDs = []string{
 		"eventName_contractAddr",

pkg/contract_watcher/header/repository/repository_suite_test.go

@@ -27,7 +27,7 @@ import (
 
 func TestRepository(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Light Repository Suite Test")
+	RunSpecs(t, "Header Sync Repository Suite Test")
 }
 
 var _ = BeforeSuite(func() {

pkg/contract_watcher/header/retriever/block_retriever_test.go

@@ -20,7 +20,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/retriever"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/retriever"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"

pkg/contract_watcher/header/retriever/retriever_suite_test.go

@@ -27,7 +27,7 @@ import (
 
 func TestRetriever(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Light Block Number Retriever Suite Test")
+	RunSpecs(t, "Header Sync Block Number Retriever Suite Test")
 }
 
 var _ = BeforeSuite(func() {

pkg/contract_watcher/header/transformer/transformer.go

@@ -24,10 +24,10 @@ import (
 	gethTypes "github.com/ethereum/go-ethereum/core/types"
 
 	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/converter"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/converter"
-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/fetcher"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/fetcher"
-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository"
-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/retriever"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/retriever"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/poller"
@@ -37,7 +37,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
-// Requires a light synced vDB (headers) and a running eth node (or infura)
+// Requires a header synced vDB (headers) and a running eth node (or infura)
 type Transformer struct {
 	// Database interfaces
 	EventRepository  srep.EventRepository        // Holds transformed watched event log data
@@ -77,14 +77,14 @@ type Transformer struct {
 func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.DB) *Transformer {
 
 	return &Transformer{
-		Poller:           poller.NewPoller(bc, db, types.LightSync),
+		Poller:           poller.NewPoller(bc, db, types.HeaderSync),
 		Fetcher:          fetcher.NewFetcher(bc),
 		Parser:           parser.NewParser(con.Network),
 		HeaderRepository: repository.NewHeaderRepository(db),
 		Retriever:        retriever.NewBlockRetriever(db),
 		Converter:        &converter.Converter{},
 		Contracts:        map[string]*contract.Contract{},
-		EventRepository:  srep.NewEventRepository(db, types.LightSync),
+		EventRepository:  srep.NewEventRepository(db, types.HeaderSync),
 		Config:           con,
 	}
 }

pkg/contract_watcher/header/transformer/transformer_suite_test.go

@@ -27,7 +27,7 @@ import (
 
 func TestTransformer(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Light Transformer Suite Test")
+	RunSpecs(t, "Header Sync Transformer Suite Test")
 }
 
 var _ = BeforeSuite(func() {

pkg/contract_watcher/header/transformer/transformer_test.go

@@ -20,8 +20,8 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/retriever"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/retriever"
-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser"
@@ -33,7 +33,7 @@ var _ = Describe("Transformer", func() {
 	var fakeAddress = "0x1234567890abcdef"
 	Describe("Init", func() {
 		It("Initializes transformer's contract objects", func() {
-			blockRetriever := &fakes.MockLightBlockRetriever{}
+			blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
 			firstBlock := int64(1)
 			blockRetriever.FirstBlock = firstBlock
 
@@ -61,7 +61,7 @@ var _ = Describe("Transformer", func() {
 		})
 
 		It("Fails to initialize if first block cannot be fetched from vDB headers table", func() {
-			blockRetriever := &fakes.MockLightBlockRetriever{}
+			blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
 			blockRetriever.FirstBlockErr = fakes.FakeError
 			t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
 
@@ -74,7 +74,7 @@ var _ = Describe("Transformer", func() {
 
 	Describe("Execute", func() {
 		It("Executes contract transformations", func() {
-			blockRetriever := &fakes.MockLightBlockRetriever{}
+			blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
 			firstBlock := int64(1)
 			blockRetriever.FirstBlock = firstBlock
 
@@ -102,7 +102,7 @@ var _ = Describe("Transformer", func() {
 		})
 
 		It("Fails to initialize if first block cannot be fetched from vDB headers table", func() {
-			blockRetriever := &fakes.MockLightBlockRetriever{}
+			blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
 			blockRetriever.FirstBlockErr = fakes.FakeError
 			t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
 
@@ -119,7 +119,7 @@ func getFakeTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Pa
 		Parser:           parsr,
 		Retriever:        blockRetriever,
 		Poller:           pollr,
-		HeaderRepository: &fakes.MockLightHeaderRepository{},
+		HeaderRepository: &fakes.MockHeaderSyncHeaderRepository{},
 		Contracts:        map[string]*contract.Contract{},
 		Config:           mocks.MockConfig,
 	}

pkg/contract_watcher/shared/helpers/test_helpers/database.go

@@ -59,7 +59,7 @@ type NewOwnerLog struct {
 	Owner          string `db:"owner_"`
 }
 
-type LightTransferLog struct {
+type HeaderSyncTransferLog struct {
 	Id        int64  `db:"id"`
 	HeaderID  int64  `db:"header_id"`
 	TokenName string `db:"token_name"`
@@ -71,7 +71,7 @@ type LightTransferLog struct {
 	RawLog    []byte `db:"raw_log"`
 }
 
-type LightNewOwnerLog struct {
+type HeaderSyncNewOwnerLog struct {
 	Id        int64  `db:"id"`
 	HeaderID  int64  `db:"header_id"`
 	TokenName string `db:"token_name"`
@@ -240,13 +240,13 @@ func TearDown(db *postgres.DB) {
 	_, err = tx.Exec(`DELETE FROM full_sync_transactions`)
 	Expect(err).NotTo(HaveOccurred())
 
-	_, err = tx.Exec("DELETE FROM light_sync_transactions")
+	_, err = tx.Exec("DELETE FROM header_sync_transactions")
 	Expect(err).NotTo(HaveOccurred())
 
 	_, err = tx.Exec(`DELETE FROM full_sync_receipts`)
 	Expect(err).NotTo(HaveOccurred())
 
-	_, err = tx.Exec(`DELETE FROM light_sync_receipts`)
+	_, err = tx.Exec(`DELETE FROM header_sync_receipts`)
 	Expect(err).NotTo(HaveOccurred())
 
 	_, err = tx.Exec(`DROP TABLE checked_headers`)
@@ -258,13 +258,13 @@ func TearDown(db *postgres.DB) {
 	_, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)
 	Expect(err).NotTo(HaveOccurred())
 
-	_, err = tx.Exec(`DROP SCHEMA IF EXISTS light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)
+	_, err = tx.Exec(`DROP SCHEMA IF EXISTS header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)
 	Expect(err).NotTo(HaveOccurred())
 
 	_, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x314159265dd8dbb310642f98f50c066173c1259b CASCADE`)
 	Expect(err).NotTo(HaveOccurred())
 
-	_, err = tx.Exec(`DROP SCHEMA IF EXISTS light_0x314159265dd8dbb310642f98f50c066173c1259b CASCADE`)
+	_, err = tx.Exec(`DROP SCHEMA IF EXISTS header_0x314159265dd8dbb310642f98f50c066173c1259b CASCADE`)
 	Expect(err).NotTo(HaveOccurred())
 
 	err = tx.Commit()

pkg/contract_watcher/shared/repository/event_repository.go

@@ -84,8 +84,8 @@ func (r *eventRepository) PersistLogs(logs []types.Log, eventInfo types.Event, c
 func (r *eventRepository) persistLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
 	var err error
 	switch r.mode {
-	case types.LightSync:
+	case types.HeaderSync:
-		err = r.persistLightSyncLogs(logs, eventInfo, contractAddr, contractName)
+		err = r.persistHeaderSyncLogs(logs, eventInfo, contractAddr, contractName)
 	case types.FullSync:
 		err = r.persistFullSyncLogs(logs, eventInfo, contractAddr, contractName)
 	default:
@@ -95,8 +95,8 @@ func (r *eventRepository) persistLogs(logs []types.Log, eventInfo types.Event, c
 	return err
 }
 
-// Creates a custom postgres command to persist logs for the given event (compatible with light synced vDB)
+// Creates a custom postgres command to persist logs for the given event (compatible with header synced vDB)
-func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
+func (r *eventRepository) persistHeaderSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
 	tx, err := r.db.Beginx()
 	if err != nil {
 		return err
@@ -232,7 +232,7 @@ func (r *eventRepository) newEventTable(tableID string, event types.Event) error
 			pgStr = pgStr + fmt.Sprintf(" %s_ %s NOT NULL,", strings.ToLower(field.Name), field.PgType)
 		}
 		pgStr = pgStr + " CONSTRAINT log_index_fk FOREIGN KEY (vulcanize_log_id) REFERENCES logs (id) ON DELETE CASCADE)"
-	case types.LightSync:
+	case types.HeaderSync:
 		pgStr = pgStr + "(id SERIAL, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, token_name CHARACTER VARYING(66) NOT NULL, raw_log JSONB, log_idx INTEGER NOT NULL, tx_idx INTEGER NOT NULL,"
 
 		for _, field := range event.Fields {

pkg/contract_watcher/shared/repository/event_repository_test.go

@@ -27,8 +27,8 @@ import (
 	. "github.com/onsi/gomega"
 
 	fc "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/converter"
-	lc "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/converter"
+	lc "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/converter"
-	lr "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository"
+	lr "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
@@ -208,9 +208,9 @@ var _ = Describe("Repository", func() {
 		})
 	})
 
-	Describe("Light sync mode", func() {
+	Describe("Header sync mode", func() {
 		BeforeEach(func() {
-			dataStore = sr.NewEventRepository(db, types.LightSync)
+			dataStore = sr.NewEventRepository(db, types.HeaderSync)
 		})
 
 		Describe("CreateContractSchema", func() {
@@ -242,7 +242,7 @@ var _ = Describe("Repository", func() {
 				Expect(err).ToNot(HaveOccurred())
 				Expect(created).To(Equal(true))
 
-				tableID := fmt.Sprintf("%s_%s.%s_event", types.LightSync, strings.ToLower(con.Address), strings.ToLower(event.Name))
+				tableID := fmt.Sprintf("%s_%s.%s_event", types.HeaderSync, strings.ToLower(con.Address), strings.ToLower(event.Name))
 				_, ok := dataStore.CheckTableCache(tableID)
 				Expect(ok).To(Equal(false))
 
@@ -292,12 +292,12 @@ var _ = Describe("Repository", func() {
 				Expect(err).ToNot(HaveOccurred())
 
 				var count int
-				err = db.Get(&count, fmt.Sprintf("SELECT COUNT(*) FROM light_%s.transfer_event", constants.TusdContractAddress))
+				err = db.Get(&count, fmt.Sprintf("SELECT COUNT(*) FROM header_%s.transfer_event", constants.TusdContractAddress))
 				Expect(err).ToNot(HaveOccurred())
 				Expect(count).To(Equal(2))
 
-				scanLog := test_helpers.LightTransferLog{}
+				scanLog := test_helpers.HeaderSyncTransferLog{}
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event LIMIT 1", constants.TusdContractAddress)).StructScan(&scanLog)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event LIMIT 1", constants.TusdContractAddress)).StructScan(&scanLog)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(scanLog.HeaderID).To(Equal(headerID))
 				Expect(scanLog.TokenName).To(Equal("TrueUSD"))
@@ -334,7 +334,7 @@ var _ = Describe("Repository", func() {
 
 				// Show that no new logs were entered
 				var count int
-				err = db.Get(&count, fmt.Sprintf("SELECT COUNT(*) FROM light_%s.transfer_event", constants.TusdContractAddress))
+				err = db.Get(&count, fmt.Sprintf("SELECT COUNT(*) FROM header_%s.transfer_event", constants.TusdContractAddress))
 				Expect(err).ToNot(HaveOccurred())
 				Expect(count).To(Equal(2))
 			})

pkg/contract_watcher/shared/repository/method_repository_test.go

@@ -149,9 +149,9 @@ var _ = Describe("Repository", func() {
 		})
 	})
 
-	Describe("Light Sync Mode", func() {
+	Describe("Header Sync Mode", func() {
 		BeforeEach(func() {
-			dataStore = repository.NewMethodRepository(db, types.LightSync)
+			dataStore = repository.NewMethodRepository(db, types.HeaderSync)
 		})
 
 		Describe("CreateContractSchema", func() {
@@ -199,7 +199,7 @@ var _ = Describe("Repository", func() {
 				Expect(err).ToNot(HaveOccurred())
 				Expect(created).To(Equal(true))
 
-				tableID := fmt.Sprintf("%s_%s.%s_method", types.LightSync, strings.ToLower(con.Address), strings.ToLower(method.Name))
+				tableID := fmt.Sprintf("%s_%s.%s_method", types.HeaderSync, strings.ToLower(con.Address), strings.ToLower(method.Name))
 				_, ok := dataStore.CheckTableCache(tableID)
 				Expect(ok).To(Equal(false))
 
@@ -214,13 +214,13 @@ var _ = Describe("Repository", func() {
 		})
 
 		Describe("PersistResult", func() {
-			It("Persists result from method polling in custom pg table for light sync mode vDB", func() {
+			It("Persists result from method polling in custom pg table for header sync mode vDB", func() {
 				err = dataStore.PersistResults([]types.Result{mockResult}, method, con.Address, con.Name)
 				Expect(err).ToNot(HaveOccurred())
 
 				scanStruct := test_helpers.BalanceOf{}
 
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method", constants.TusdContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method", constants.TusdContractAddress)).StructScan(&scanStruct)
 				expectedLog := test_helpers.BalanceOf{
 					Id:        1,
 					TokenName: "TrueUSD",

pkg/contract_watcher/shared/types/event.go

@@ -38,14 +38,14 @@ type Field struct {
 
 // Struct to hold instance of an event log data
 type Log struct {
-	Id     int64             // VulcanizeIdLog for full sync and header ID for light sync contract watcher
+	Id     int64             // VulcanizeIdLog for full sync and header ID for header sync contract watcher
 	Values map[string]string // Map of event input names to their values
 
 	// Used for full sync only
 	Block int64
 	Tx    string
 
-	// Used for lightSync only
+	// Used for headerSync only
 	LogIndex         uint
 	TransactionIndex uint
 	Raw              []byte // json.Unmarshalled byte array of geth/core/types.Log{}

pkg/contract_watcher/shared/types/mode.go

@@ -21,18 +21,18 @@ import "fmt"
 type Mode int
 
 const (
-	LightSync Mode = iota
+	HeaderSync Mode = iota
 	FullSync
 )
 
 func (mode Mode) IsValid() bool {
-	return mode >= LightSync && mode <= FullSync
+	return mode >= HeaderSync && mode <= FullSync
 }
 
 func (mode Mode) String() string {
 	switch mode {
-	case LightSync:
+	case HeaderSync:
-		return "light"
+		return "header"
 	case FullSync:
 		return "full"
 	default:
@@ -42,23 +42,23 @@ func (mode Mode) String() string {
 
 func (mode Mode) MarshalText() ([]byte, error) {
 	switch mode {
-	case LightSync:
+	case HeaderSync:
-		return []byte("light"), nil
+		return []byte("header"), nil
 	case FullSync:
 		return []byte("full"), nil
 	default:
-		return nil, fmt.Errorf("contract watcher: unknown mode %d, want LightSync or FullSync", mode)
+		return nil, fmt.Errorf("contract watcher: unknown mode %d, want HeaderSync or FullSync", mode)
 	}
 }
 
 func (mode *Mode) UnmarshalText(text []byte) error {
 	switch string(text) {
-	case "light":
+	case "header":
-		*mode = LightSync
+		*mode = HeaderSync
 	case "full":
 		*mode = FullSync
 	default:
-		return fmt.Errorf(`contract watcher: unknown mode %q, want "light" or "full"`, text)
+		return fmt.Errorf(`contract watcher: unknown mode %q, want "header" or "full"`, text)
 	}
 	return nil
 }

pkg/datastore/postgres/repositories/header_repository.go

@@ -54,7 +54,7 @@ func (repository HeaderRepository) CreateOrUpdateHeader(header core.Header) (int
 
 func (repository HeaderRepository) CreateTransactions(headerID int64, transactions []core.TransactionModel) error {
 	for _, transaction := range transactions {
-		_, err := repository.database.Exec(`INSERT INTO public.light_sync_transactions
+		_, err := repository.database.Exec(`INSERT INTO public.header_sync_transactions
 		(header_id, hash, gas_limit, gas_price, input_data, nonce, raw, tx_from, tx_index, tx_to, "value") 
 		VALUES ($1, $2, $3::NUMERIC, $4::NUMERIC, $5, $6::NUMERIC, $7, $8, $9::NUMERIC, $10, $11::NUMERIC)
 		ON CONFLICT DO NOTHING`, headerID, transaction.Hash, transaction.GasLimit, transaction.GasPrice,
@@ -69,7 +69,7 @@ func (repository HeaderRepository) CreateTransactions(headerID int64, transactio
 
 func (repository HeaderRepository) CreateTransactionInTx(tx *sqlx.Tx, headerID int64, transaction core.TransactionModel) (int64, error) {
 	var txId int64
-	err := tx.QueryRowx(`INSERT INTO public.light_sync_transactions
+	err := tx.QueryRowx(`INSERT INTO public.header_sync_transactions
 		(header_id, hash, gas_limit, gas_price, input_data, nonce, raw, tx_from, tx_index, tx_to, "value")
 		VALUES ($1, $2, $3::NUMERIC, $4::NUMERIC, $5, $6::NUMERIC, $7, $8, $9::NUMERIC, $10, $11::NUMERIC)
 		ON CONFLICT (header_id, hash) DO UPDATE 
@@ -87,7 +87,7 @@ func (repository HeaderRepository) CreateTransactionInTx(tx *sqlx.Tx, headerID i
 
 func (repository HeaderRepository) CreateReceiptInTx(tx *sqlx.Tx, headerID, transactionID int64, receipt core.Receipt) (int64, error) {
 	var receiptId int64
-	err := tx.QueryRowx(`INSERT INTO public.light_sync_receipts
+	err := tx.QueryRowx(`INSERT INTO public.header_sync_receipts
                (header_id, transaction_id, contract_address, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp)
                VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
 			   ON CONFLICT (header_id, transaction_id) DO UPDATE

pkg/datastore/postgres/repositories/header_repository_test.go

@@ -228,7 +228,7 @@ var _ = Describe("Block header repository", func() {
 			var dbReceipt idModel
 			err = db.Get(&dbReceipt,
 				`SELECT transaction_id, contract_address, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp
-				FROM public.light_sync_receipts WHERE header_id = $1`, headerID)
+				FROM public.header_sync_receipts WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(dbReceipt.TransactionId).To(Equal(txId))
 			Expect(dbReceipt.TxHash).To(Equal(txHash.Hex()))
@@ -288,7 +288,7 @@ var _ = Describe("Block header repository", func() {
 			var dbTransactions []core.TransactionModel
 			err = db.Select(&dbTransactions,
 				`SELECT hash, gas_limit, gas_price, input_data, nonce, raw, tx_from, tx_index, tx_to, "value"
-				FROM public.light_sync_transactions WHERE header_id = $1`, headerID)
+				FROM public.header_sync_transactions WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(dbTransactions).To(ConsistOf(transactions))
 		})
@@ -300,7 +300,7 @@ var _ = Describe("Block header repository", func() {
 			var dbTransactions []core.TransactionModel
 			err = db.Select(&dbTransactions,
 				`SELECT hash, gas_limit, gas_price, input_data, nonce, raw, tx_from, tx_index, tx_to, "value"
-				FROM public.light_sync_transactions WHERE header_id = $1`, headerID)
+				FROM public.header_sync_transactions WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(dbTransactions)).To(Equal(2))
 		})
@@ -337,7 +337,7 @@ var _ = Describe("Block header repository", func() {
 			var dbTransaction core.TransactionModel
 			err = db.Get(&dbTransaction,
 				`SELECT hash, gas_limit, gas_price, input_data, nonce, raw, tx_from, tx_index, tx_to, "value"
-				FROM public.light_sync_transactions WHERE header_id = $1`, headerID)
+				FROM public.header_sync_transactions WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(dbTransaction).To(Equal(transaction))
 		})
@@ -381,7 +381,7 @@ var _ = Describe("Block header repository", func() {
 			var dbTransactions []core.TransactionModel
 			err = db.Select(&dbTransactions,
 				`SELECT hash, gas_limit, gas_price, input_data, nonce, raw, tx_from, tx_index, tx_to, "value"
-				FROM public.light_sync_transactions WHERE header_id = $1`, headerID)
+				FROM public.header_sync_transactions WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(dbTransactions)).To(Equal(1))
 		})

pkg/fakes/data.go

@@ -43,11 +43,15 @@ var FakeHeader = core.Header{
 }
 
 func GetFakeHeader(blockNumber int64) core.Header {
+	return GetFakeHeaderWithTimestamp(fakeTimestamp, blockNumber)
+}
+
+func GetFakeHeaderWithTimestamp(timestamp, blockNumber int64) core.Header {
 	return core.Header{
 		Hash:        FakeHash.String(),
 		BlockNumber: blockNumber,
 		Raw:         rawFakeHeader,
-		Timestamp:   strconv.FormatInt(fakeTimestamp, 10),
+		Timestamp:   strconv.FormatInt(timestamp, 10),
 	}
 }
 

pkg/fakes/mock_filter_repository.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package fakes
 
 import "github.com/vulcanize/vulcanizedb/pkg/filters"

pkg/fakes/mock_full_block_retriever.go

@@ -1,15 +0,0 @@
-package fakes
-
-type MockFullBlockRetriever struct {
-	FirstBlock      int64
-	FirstBlockErr   error
-	MostRecentBlock int64
-}
-
-func (retriever *MockFullBlockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
-	return retriever.FirstBlock, retriever.FirstBlockErr
-}
-
-func (retriever *MockFullBlockRetriever) RetrieveMostRecentBlock() (int64, error) {
-	return retriever.MostRecentBlock, nil
-}

pkg/fakes/mock_full_sync_block_retriever.go

@@ -0,0 +1,15 @@
+package fakes
+
+type MockFullSyncBlockRetriever struct {
+	FirstBlock      int64
+	FirstBlockErr   error
+	MostRecentBlock int64
+}
+
+func (retriever *MockFullSyncBlockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
+	return retriever.FirstBlock, retriever.FirstBlockErr
+}
+
+func (retriever *MockFullSyncBlockRetriever) RetrieveMostRecentBlock() (int64, error) {
+	return retriever.MostRecentBlock, nil
+}

pkg/fakes/mock_header_sync_block_retriever.go

@@ -0,0 +1,14 @@
+package fakes
+
+type MockHeaderSyncBlockRetriever struct {
+	FirstBlock    int64
+	FirstBlockErr error
+}
+
+func (retriever *MockHeaderSyncBlockRetriever) RetrieveFirstBlock() (int64, error) {
+	return retriever.FirstBlock, retriever.FirstBlockErr
+}
+
+func (retriever *MockHeaderSyncBlockRetriever) RetrieveMostRecentBlock() (int64, error) {
+	return 0, nil
+}

pkg/fakes/mock_header_sync_header_repository.go

@@ -0,0 +1,42 @@
+package fakes
+
+import "github.com/vulcanize/vulcanizedb/pkg/core"
+
+type MockHeaderSyncHeaderRepository struct {
+}
+
+func (*MockHeaderSyncHeaderRepository) AddCheckColumn(id string) error {
+	return nil
+}
+
+func (*MockHeaderSyncHeaderRepository) AddCheckColumns(ids []string) error {
+	panic("implement me")
+}
+
+func (*MockHeaderSyncHeaderRepository) MarkHeaderChecked(headerID int64, eventID string) error {
+	panic("implement me")
+}
+
+func (*MockHeaderSyncHeaderRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) error {
+	panic("implement me")
+}
+
+func (*MockHeaderSyncHeaderRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error {
+	panic("implement me")
+}
+
+func (*MockHeaderSyncHeaderRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, eventID string) ([]core.Header, error) {
+	panic("implement me")
+}
+
+func (*MockHeaderSyncHeaderRepository) MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error) {
+	panic("implement me")
+}
+
+func (*MockHeaderSyncHeaderRepository) MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error) {
+	panic("implement me")
+}
+
+func (*MockHeaderSyncHeaderRepository) CheckCache(key string) (interface{}, bool) {
+	panic("implement me")
+}

pkg/fakes/mock_light_block_retriever.go

@@ -1,14 +0,0 @@
-package fakes
-
-type MockLightBlockRetriever struct {
-	FirstBlock    int64
-	FirstBlockErr error
-}
-
-func (retriever *MockLightBlockRetriever) RetrieveFirstBlock() (int64, error) {
-	return retriever.FirstBlock, retriever.FirstBlockErr
-}
-
-func (retriever *MockLightBlockRetriever) RetrieveMostRecentBlock() (int64, error) {
-	return 0, nil
-}

pkg/fakes/mock_light_header_repository.go

@@ -1,42 +0,0 @@
-package fakes
-
-import "github.com/vulcanize/vulcanizedb/pkg/core"
-
-type MockLightHeaderRepository struct {
-}
-
-func (*MockLightHeaderRepository) AddCheckColumn(id string) error {
-	return nil
-}
-
-func (*MockLightHeaderRepository) AddCheckColumns(ids []string) error {
-	panic("implement me")
-}
-
-func (*MockLightHeaderRepository) MarkHeaderChecked(headerID int64, eventID string) error {
-	panic("implement me")
-}
-
-func (*MockLightHeaderRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) error {
-	panic("implement me")
-}
-
-func (*MockLightHeaderRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error {
-	panic("implement me")
-}
-
-func (*MockLightHeaderRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, eventID string) ([]core.Header, error) {
-	panic("implement me")
-}
-
-func (*MockLightHeaderRepository) MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error) {
-	panic("implement me")
-}
-
-func (*MockLightHeaderRepository) MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error) {
-	panic("implement me")
-}
-
-func (*MockLightHeaderRepository) CheckCache(key string) (interface{}, bool) {
-	panic("implement me")
-}

pkg/fakes/mock_parser.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package fakes
 
 import (

pkg/fakes/mock_poller.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package fakes
 
 import (

pkg/fakes/mock_tailer.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package fakes
 
 import (
@@ -6,24 +22,22 @@ import (
 )
 
 type MockTailer struct {
-	Lines      chan *tail.Line
+	Lines   chan *tail.Line
-	TailCalled bool
+	TailErr error
 }
 
 func NewMockTailer() *MockTailer {
 	return &MockTailer{
-		Lines:      make(chan *tail.Line, 1),
+		Lines: make(chan *tail.Line, 1),
-		TailCalled: false,
 	}
 }
 
 func (mock *MockTailer) Tail() (*tail.Tail, error) {
-	mock.TailCalled = true
 	fakeTail := &tail.Tail{
 		Filename: "",
 		Lines:    mock.Lines,
 		Config:   tail.Config{},
 		Tomb:     tomb.Tomb{},
 	}
-	return fakeTail, nil
+	return fakeTail, mock.TailErr
 }

pkg/fakes/mock_transaction_syncer.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package fakes
 
 import "github.com/ethereum/go-ethereum/core/types"

pkg/geth/blockchain.go

@@ -18,7 +18,6 @@ package geth
 
 import (
 	"errors"
-	"fmt"
 	"github.com/ethereum/go-ethereum"
 	"math/big"
 	"strconv"
@@ -122,7 +121,6 @@ func (blockChain *BlockChain) GetTransactions(transactionHashes []common.Hash) (
 
 	rpcErr := blockChain.rpcClient.BatchCall(batch)
 	if rpcErr != nil {
-		fmt.Println("rpc err")
 		return []core.TransactionModel{}, rpcErr
 	}
 

pkg/geth/converters/rpc/transaction_converter.go

@@ -78,7 +78,7 @@ func (converter *RpcTransactionConverter) ConvertRpcTransactionsToModels(transac
 			Hash:     transaction.Hash,
 			Nonce:    txData.AccountNonce,
 			Raw:      txRLP,
-			// NOTE: Light Sync transactions don't include receipt; would require separate RPC call
+			// NOTE: Header Sync transactions don't include receipt; would require separate RPC call
 			To:      transaction.Recipient,
 			TxIndex: txIndex.Int64(),
 			Value:   txData.Amount.String(),

pkg/plugin/manager/manager.go

@@ -128,7 +128,13 @@ func (m *manager) fixAndRun(path string) error {
 		return errors.New(fmt.Sprintf("version fixing for plugin migrations at %s failed: %s", path, err.Error()))
 	}
 	// Run the copied migrations with goose
-	pgStr := fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable", m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name)
+	var pgStr string
+	if len(m.DBConfig.User) > 0 && len(m.DBConfig.Password) > 0 {
+		pgStr = fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable",
+			m.DBConfig.User, m.DBConfig.Password, m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name)
+	} else {
+		pgStr = fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable", m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name)
+	}
 	cmd = exec.Command("goose", "postgres", pgStr, "up")
 	cmd.Dir = m.tmpMigDir
 	err = cmd.Run()

postgraphile/README.md

@@ -9,10 +9,32 @@ Build the docker image in this directory. Start the `GraphiQL` frontend by:
 * Setting the env variables for the database connection: `DATABASE_HOST`,
   `DATABASE_NAME`, `DATABASE_USER`, `DATABASE_PASSWORD` (and optionally
   `DATABASE_PORT` if running on non-standard port).
-  * The specified user needs to be `superuser` on the vulcanizeDB database
+  * The specified user needs to be `superuser` on the vulcanizeDB database,
-* Run the container (ex. `docker run -e DATABASE_HOST=localhost -e DATABASE_NAME=vulcanize_public -e DATABASE_USER=vulcanize -e DATABASE_PASSWORD=vulcanize -d m0ar/images:postgraphile-alpine`)
+    so postgraphile can setup watch fixtures keeping track of live schema
-* GraphiQL is available at `:3000/graphiql`
+    changes.
+* To limit the amount of available queries in GraphQL, a restricted user can be used
+  for postgraphile introspection by adding env variables `GQ_USER` and `GQ_PASSWORD`.
+  * By doing `GRANT [SELECT | EXECUTE]` on tables/functions for this user,
+    you can selectively assign things you want available in GraphQL.
+  * You still need to pass in a superuser with `DATABASE_USER` & `DATABASE_PASSWORD` for
+    the postgraphile watch fixtures to work.
+* By default, postgraphile publishes the `public` schema. This can be expanded with for example `GQ_SCHEMAS=public,maker`
+* Run the container (ex. `docker run -e DATABASE_HOST=localhost -e DATABASE_NAME=my_database -e DATABASE_USER=superuser -e DATABASE_PASSWORD=superuser -e GQ_USER=graphql -e GQ_PASSWORD=graphql -e GQ_SCHEMAS=public,anotherSchema -d my-postgraphile-image`)
+* GraphiQL frontend is available at `:3000/graphiql`
+  GraphQL endpoint is available at `:3000/graphql`
 
+By default, this build will expose only the "public" schema and will disable mutations - to change mutation behaviour, you can use an optional config file `config.toml` and set the env var `POSTGRAPHILE_CONFIG_PATH` to point to its location. Example `toml`:
+
+```
+[database]
+    name     = "vulcanize_public"
+    hostname = "localhost"
+    port = 5432
+    gq_schemas = ["public", "yourschema"]
+    gq_user = "graphql"
+    gq_password = "graphql"
+    disable_default_mutations = false
+```
 
 ## Building
 
@@ -20,8 +42,6 @@ Build the docker image in this directory. Start the `GraphiQL` frontend by:
 
 Install dependencies with `yarn` and execute `yarn build`. The bundle produced by Webpack will be present in `build/dist/`.
 
-This application currently uses the Postgraphile supporter plugin. This plugin is present in the `vendor/` directory and is copied to `node_modules/` after installation of packages. It is a fresh checkout of the plugin as of August 31st, 2018.
-
 ## Running
 
 Provide the built bundle to node as a runnable script: `node ./build/dist/vulcanize-postgraphile-server.js`

postgraphile/package.json

@@ -5,8 +5,8 @@
   "scripts": {
     "build": "rm -rf ./build/dist && webpack --config=./webpack.config.js",
     "lint": "tslint --project ./tsconfig.json --config ./tslint.json",
-    "postinstall": "rm -rf node_modules/@graphile && mkdir node_modules/@graphile && cp -R ./vendor/postgraphile-supporter/ ./node_modules/@graphile/plugin-supporter/",
     "start": "npm run build && node ./build/dist/vulcanize-postgraphile-server.js",
+    "dev": "./node_modules/typescript/bin/tsc && node build/dist/src/index.js",
     "test": "rm -rf ./build/spec && tsc --build ./tsconfig.test.json && jasmine --config=./spec/support/jasmine.json",
     "test:ci": "npm run lint && npm run test"
   },
@@ -23,11 +23,11 @@
   "dependencies": {
     "express-session": "1.15.6",
     "graphql-subscriptions": "0.5.8",
-    "lodash": "4.17.10",
+    "lodash": ">=4.17.11",
     "passport": "0.4.0",
     "pg": "6.4.2",
     "pg-native": "3.0.0",
-    "postgraphile": "4.0.0-rc.4",
+    "postgraphile": "4.4.0-beta.11",
     "subscriptions-transport-ws": "0.9.14",
     "toml": "2.3.3"
   },
@@ -39,6 +39,7 @@
     "@types/lodash": "4.14.116",
     "@types/node": "^10.12.21",
     "@types/passport": "0.4.6",
+    "@graphile-contrib/pg-simplify-inflector": "3.0.0",
     "awesome-typescript-loader": "5.2.0",
     "jasmine": "3.2.0",
     "jasmine-ts-console-reporter": "3.1.1",
@@ -48,7 +49,7 @@
     "typescript": "3.0.1",
     "webpack": "4.17.1",
     "webpack-cli": "3.1.0",
-    "webpack-dev-server": "3.1.6"
+    "webpack-dev-server": ">=3.1.11"
   },
   "resolutions": {
     "pg": "6.4.2"

postgraphile/spec/config/parse.spec.ts

@@ -33,7 +33,8 @@ describe('parseConfig', () => {
     const databaseConfig = parseConfig(
       readCallback, tomlParseCallback, configPath);
 
-    expect(databaseConfig.host).toEqual('postgres://user:password@example.com:1234');
+    expect(databaseConfig.host)
+      .toEqual('postgres://user:password@example.com:1234');
   });
 
   it('provides the database name', () => {

postgraphile/spec/server/config.spec.ts

@@ -21,14 +21,19 @@ describe('buildServerConfig', () => {
   let databaseConfig: DatabaseConfig;
 
   beforeEach(() => {
-    databaseConfig = { host: 'example.com', database: 'example_database' };
+    databaseConfig = {
+      host: 'example.com',
+      database: 'example_database',
+      schemas: ['public'],
+      ownerConnectionString: 'postgres://admin:admin@host',
+      disableDefaultMutations: true
+    };
 
     postgraphileMiddleware = jasmine
-      .createSpyObj<PostgraphileMiddleware>(['call']),
+      .createSpyObj<PostgraphileMiddleware>(['call']);
 
     serverUtilities = {
       pluginHook: jasmine.createSpy('pluginHook'),
-      enableSubscriptions: jasmine.createSpy('enableSubscriptions'),
       express: jasmine.createSpy('express'),
       expressSession: jasmine.createSpy('expressSession'),
       httpServerFactory: jasmine.createSpy('httpServerFactory'),
@@ -62,10 +67,6 @@ describe('buildServerConfig', () => {
     expect(serverConfig.options).not.toBeNull();
   });
 
-  it('enables simple subscriptions', () => {
-    expect(serverConfig.options.simpleSubscriptions).toBe(true);
-  });
-
   it('it adds the express session handler as the first middleware', () => {
     expect(serverConfig.options.webSocketMiddlewares[0])
       .toBe(expressSessionHandler);
@@ -84,7 +85,7 @@ describe('buildServerConfig', () => {
   it('provides the database config to Postgraphile', () => {
     expect(serverUtilities.postgraphile).toHaveBeenCalledWith(
       `${databaseConfig.host}/${databaseConfig.database}`,
-      ["public"],
+      databaseConfig.schemas,
       jasmine.any(Object));
   });
 

postgraphile/spec/server/runtime.spec.ts

@@ -15,7 +15,6 @@ describe('bootServer', () => {
   beforeEach(() => {
     serverUtilities = {
       pluginHook: jasmine.createSpy('pluginHook'),
-      enableSubscriptions: jasmine.createSpy('enableSubscriptions'),
       express: jasmine.createSpy('express'),
       expressSession: jasmine.createSpy('expressSession'),
       httpServerFactory: jasmine.createSpy('httpServerFactory'),
@@ -26,11 +25,15 @@ describe('bootServer', () => {
     serverConfig = {
       middleware: jasmine.createSpyObj<PostgraphileMiddleware>(['call']),
       options: {
+        appendPlugins: [],
+        disableDefaultMutations: false,
+        enableCors: true,
+        exportGqlSchemaPath: '',
+        graphiql: true,
+        ignoreRBAC: false,
+        ownerConnectionString: '',
         pluginHook: jasmine.createSpy('pluginHook'),
         watchPg: true,
-        enableCors: true,
-        simpleSubscriptions: true,
-        graphiql: true,
         webSocketMiddlewares: [] },
       port: 5678
     };
@@ -56,14 +59,6 @@ describe('bootServer', () => {
     expect(useSpy).toHaveBeenCalledWith(serverConfig.middleware);
   });
 
-  it('enahances the Node HTTP server with Postgraphile subscriptions', () => {
-    expect(serverUtilities.enableSubscriptions)
-      .toHaveBeenCalledWith(
-        mockHttpServer,
-        serverConfig.middleware,
-        serverConfig.options);
-  });
-
   it('instructs the server to listen on the given port', () => {
     const listenSpy = mockHttpServer.listen as jasmine.Spy;
     expect(listenSpy).toHaveBeenCalledWith(serverConfig.port);

postgraphile/src/adapters/fs.ts

@@ -8,4 +8,4 @@ export type ReadFileSyncCallback = (
 ) => string | Buffer;
 
 export type TomlParseCallback
-  = (fileContents: string) => { [key: string]: { [key: string]: string } };
+  = (fileContents: string) => { [key: string]: { [key: string]: any } };

postgraphile/src/adapters/postgraphile.ts

@@ -1,6 +1,6 @@
 import { RequestHandler } from 'express';
-import { Server } from 'http';
+import {PluginHookFn } from 'postgraphile/build/postgraphile/pluginHook';
-import { PluginHookFn } from 'postgraphile/build/postgraphile/pluginHook';
+import {Plugin} from 'postgraphile';
 
 // NOTE: Shape of the middleware is not
 // currently important to this application, but if a need arises,
@@ -9,12 +9,16 @@ import { PluginHookFn } from 'postgraphile/build/postgraphile/pluginHook';
 export interface PostgraphileMiddleware extends RequestHandler {}
 
 export interface PostgraphileOptions {
-  pluginHook: PluginHookFn,
+  appendPlugins: Plugin[];
-  simpleSubscriptions: boolean;
+  disableDefaultMutations: boolean;
-  watchPg: boolean;
   enableCors: boolean;
+  exportGqlSchemaPath: string;
   graphiql: boolean;
-  // NOTE: Shape of the middlewares is not
+  ignoreRBAC: boolean;
+  ownerConnectionString: string;
+  pluginHook: PluginHookFn;
+  watchPg: boolean;
+  // NOTE Shape of the middlewares is not
   // currently important to this application, but if a need arises,
   // any needed shape can be assigned from a custom type here.
   webSocketMiddlewares: object[];
@@ -26,8 +30,3 @@ export type PostgraphileInitCallback = (
   options: PostgraphileOptions
 ) => PostgraphileMiddleware;
 
-export type AddSubscriptionsCallback = (
-  httpServer: Server,
-  middleware: PostgraphileMiddleware,
-  options: PostgraphileOptions
-) => void;

postgraphile/src/config/parse.ts

@@ -6,8 +6,10 @@ export const MISSING_PATH_MESSAGE = `No path to config toml file provided, `
   + `please check the value of ${CONFIG_PATH_KEY} in your environment`;
 
 export const MISSING_HOST_MESSAGE = 'No database host provided in config toml';
-export const MISSING_USER_MESSAGE = 'No database user & password provided in config toml';
+export const MISSING_USER_MESSAGE = 'No database user & password '
-export const MISSING_DATABASE_MESSAGE = 'No database name provided in config toml';
+  + 'provided in config toml';
+export const MISSING_DATABASE_MESSAGE = 'No database name provided '
+  + 'in config toml';
 
 export function parseConfig(
   readCallback: ReadFileSyncCallback,
@@ -19,16 +21,26 @@ export function parseConfig(
   let database = '';
   let user = '';
   let password = '';
+  let gqSchemas = ['public'];
+  let gqUser = '';
+  let gqPassword = '';
+  let disableDefaultMutations = true;
 
   if (configPath) {
-      const tomlContents = readCallback(`${configPath}`).toString();
+    const tomlContents = readCallback(`${configPath}`).toString();
-      const parsedToml = tomlParseCallback(tomlContents);
+    const parsedToml = tomlParseCallback(tomlContents);
 
-      host = parsedToml['database']['hostname'];
+    host = parsedToml['database']['hostname'];
-      port = parsedToml['database']['port'];
+    port = parsedToml['database']['port'];
-      database = parsedToml['database']['name'];
+    database = parsedToml['database']['name'];
-      user = parsedToml['database']['user'];
+    user = parsedToml['database']['user'];
-      password = parsedToml['database']['password'];
+    password = parsedToml['database']['password'];
+    gqSchemas = parsedToml['database']['gq_schemas'];
+    gqUser = parsedToml['database']['gq_user'] || gqUser;
+    gqPassword = parsedToml['database']['gq_password'] || gqPassword;
+    disableDefaultMutations = parsedToml['database']['disable_default_mutations'] === undefined
+      ? true
+      : parsedToml['database']['disable_default_mutations'];
   }
 
   // Overwrite config values with env. vars if such are set
@@ -37,6 +49,11 @@ export function parseConfig(
   database = process.env.DATABASE_NAME || database;
   user = process.env.DATABASE_USER || user;
   password = process.env.DATABASE_PASSWORD || password;
+  gqSchemas = process.env.GQ_SCHEMAS
+      ? process.env.GQ_SCHEMAS.split(',')
+      : gqSchemas;
+  gqUser = process.env.GQ_USER || gqUser;
+  gqPassword = process.env.GQ_PASSWORD || gqPassword;
 
   if (!host) {
     throw new Error(MISSING_HOST_MESSAGE);
@@ -50,5 +67,13 @@ export function parseConfig(
     throw new Error(MISSING_USER_MESSAGE);
   }
 
-  return { host: `postgres://${user}:${password}@${host}:${port}`, database };
+  return {
+    host: gqUser && gqPassword
+        ? `postgres://${gqUser}:${gqPassword}@${host}:${port}`
+        : `postgres://${user}:${password}@${host}:${port}`,
+    database,
+    schemas: gqSchemas,
+    ownerConnectionString: `postgres://${user}:${password}@${host}:${port}/${database}`,
+    disableDefaultMutations
+  };
 }

postgraphile/src/index.ts

@@ -7,11 +7,7 @@ import passport = require('passport');
 import session = require('express-session');
 import toml = require('toml');
 
-const {
+const pluginHook = makePluginHook([]);
-  default: PostGraphileSupporter,
-  enhanceHttpServerWithSubscriptions,
-} = require('@graphile/plugin-supporter');
-const pluginHook = makePluginHook([PostGraphileSupporter]);
 
 import { ServerUtilities } from './server/interface';
 import { bootServer } from './server/runtime';
@@ -27,7 +23,6 @@ const configPath = process.env[CONFIG_PATH_KEY];
 const serverPort = process.env[SERVER_PORT_KEY];
 
 const serverUtilities: ServerUtilities = {
-  enableSubscriptions: enhanceHttpServerWithSubscriptions,
   express,
   expressSession: session,
   httpServerFactory: createServer,

postgraphile/src/server/config.ts

@@ -5,7 +5,7 @@ import {
   PostgraphileOptions
 } from '../adapters/postgraphile';
 
-export const CONFIG_PATH_KEY = 'CONFIG_PATH';
+export const CONFIG_PATH_KEY = 'POSTGRAPHILE_CONFIG_PATH';
 export const SERVER_PORT_KEY = 'SERVER_PORT';
 
 const DEFAULT_SERVER_PORT = '3000';
@@ -23,13 +23,18 @@ export function buildServerConfig(
   const passportInitializer = utilities.passport.initialize();
   const passportSessionHandler = utilities.passport.session();
   const pluginHook = utilities.pluginHook;
+  const PgSimplifyInflectorPlugin = require('@graphile-contrib/pg-simplify-inflector');
 
   const options: PostgraphileOptions = {
-    pluginHook: pluginHook,
+    appendPlugins: [PgSimplifyInflectorPlugin],
-    simpleSubscriptions: true,
+    disableDefaultMutations: databaseConfig.disableDefaultMutations,
-    watchPg: true,
     enableCors: true,
+    exportGqlSchemaPath: 'schema.graphql',
     graphiql: true,
+    ignoreRBAC: false,
+    ownerConnectionString: databaseConfig.ownerConnectionString,
+    pluginHook: pluginHook,
+    watchPg: true,
     webSocketMiddlewares: [
       expressSessionHandler,
       passportInitializer,
@@ -39,7 +44,7 @@ export function buildServerConfig(
 
   const middleware: PostgraphileMiddleware = utilities.postgraphile(
     `${databaseConfig.host}/${databaseConfig.database}`,
-    ["public"],
+    databaseConfig.schemas,
     options
   );
 

postgraphile/src/server/interface.ts

@@ -7,7 +7,6 @@ import {
 } from '../adapters/session';
 
 import {
-  AddSubscriptionsCallback,
   PostgraphileInitCallback,
   PostgraphileMiddleware,
   PostgraphileOptions
@@ -17,6 +16,9 @@ import { PluginHookFn } from 'postgraphile/build/postgraphile/pluginHook';
 export interface DatabaseConfig {
   host: string;
   database: string;
+  schemas: string[];
+  ownerConnectionString: string;
+  disableDefaultMutations: boolean;
 }
 
 export interface ServerConfig {
@@ -26,11 +28,10 @@ export interface ServerConfig {
 }
 
 export interface ServerUtilities {
-  enableSubscriptions: AddSubscriptionsCallback;
   express: ExpressInitCallback;
   expressSession: ExpressSessionInitCallback;
   httpServerFactory: CreateHttpServerCallback;
   passport: StaticPassportProvider;
   postgraphile: PostgraphileInitCallback;
-  pluginHook: PluginHookFn
+  pluginHook: PluginHookFn;
 }

postgraphile/src/server/runtime.ts

@@ -9,10 +9,5 @@ export function bootServer(
 
   const httpServer = utilities.httpServerFactory(expressApp);
   
-  utilities.enableSubscriptions(
-    httpServer,
-    config.middleware,
-    config.options);
-  
   httpServer.listen(config.port);
 }

test_config/test_config.go

@@ -111,12 +111,12 @@ func CleanTestDB(db *postgres.DB) {
 	db.MustExec("DELETE FROM full_sync_transactions")
 	db.MustExec("DELETE FROM goose_db_version")
 	db.MustExec("DELETE FROM headers")
-	db.MustExec("DELETE FROM light_sync_transactions")
+	db.MustExec("DELETE FROM header_sync_transactions")
 	db.MustExec("DELETE FROM log_filters")
 	db.MustExec("DELETE FROM logs")
 	db.MustExec("DELETE FROM queued_storage")
 	db.MustExec("DELETE FROM full_sync_receipts")
-	db.MustExec("DELETE FROM light_sync_receipts")
+	db.MustExec("DELETE FROM header_sync_receipts")
 	db.MustExec("DELETE FROM watched_contracts")
 }