Merge pull request #2 from vulcanize/watch

finish updating documentation after refactor

.gitignore

@@ -7,7 +7,7 @@ Vagrantfile
 vagrant*.sh
 .vagrant
 test_scripts/
-vulcanizedb
+ipfs-chain-watcher
 postgraphile/build/
 postgraphile/node_modules/
 postgraphile/package-lock.json
@@ -16,3 +16,4 @@ db/migrations/20*.sql
 plugins/*.so
 postgraphile/*.toml
 postgraphile/schema.graphql
+vulcanizedb.pem

.travis.yml

@@ -1,26 +1,27 @@
 dist: trusty
 language: go
 go:
-  - 1.12
+- 1.12
 services:
-  - postgresql
+- postgresql
 addons:
+  ssh_known_hosts: arch1.vdb.to
   postgresql: '11.2'
-go_import_path: github.com/vulcanize/vulcanizedb
+go_import_path: github.com/vulcanize/ipfs-chain-watcher
 before_install:
-  - make installtools
-  - bash ./scripts/install-postgres-11.sh
-  - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
-  - echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
-  - sudo apt-get update && sudo apt-get install yarn
-before_script:
-  - sudo -u postgres createdb vulcanize_private
-  - make migrate NAME=vulcanize_private
+- openssl aes-256-cbc -K $encrypted_e1db309e8776_key -iv $encrypted_e1db309e8776_iv
+  -in temp_rsa.enc -out temp_rsa -d
+- eval "$(ssh-agent -s)"
+- chmod 600 temp_rsa
+- ssh-add temp_rsa
+- ssh -4 -fNL 8545:localhost:8545 geth@arch1.vdb.to
+- make installtools
+- bash ./scripts/install-postgres-11.sh
+- curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
+- echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
+- sudo apt-get update && sudo apt-get install yarn
 script:
-  - env GO111MODULE=on make test
-  - env GO111MODULE=on make integrationtest
+- env GO111MODULE=on make test
+- env GO111MODULE=on make integrationtest
 notifications:
   email: false
-env:
-  matrix:
-    secure: GA4GiMHy868UD/C+59jRUOdJxxdLSMauNapPA31AODUMv0lcEOXg6TBJkv6Vn3OOrLHYYSpwCGiGQjF86U2vpKWzD1mSITalPfP/kNpFnrHVNf2oPpfrQs8ccs9gnFpbCNLDy0w2+55pgnrd5hvdnlsTEUS1lHWSKEZmWsSqUeS/dw22IS0+nUja0Pc3pHaWlJz2R502uEKIixjq1fOX65Z/9r1BrMqklOwWrABK0GwgYcDZwc+b3iovyDPkDa3sT/Ddz/srkcYi/Y182zdLdrknj3MVR3GiW5iSqIVm6gr427SFXF4EQVwkEBwAGoo83wRhCgCAslYTRZZ5aleb1HUtAWzuiMAd/JlAYb2jc6VJa19+rQ/5wd7Bann3hB5bY2uV+b02qFK3qWe6uqZLBSESWjFEyWJVoaXgpH0QDc0JG1R9UFxLSVa8NM3P/jgTnUgnAvdW36mTC4dg73Ha9bYdeLpuAZr6k8iTh5qp0CazKAazsdJ+ECNt6q3fkgsI/V5ZwCFfQwlauq+PiBaZsQqr7eeP19klipE3vGf6BCBRvCBdubzSgGZPIRkxOQREc24j/gzBjBNGeqoU2uOp8sXMEG5idfIyHbMU+n2Yi6IRKKPe5lwjgWEsqLkkoM2REMPIEPffRQz0f+mD48HWP0H4KjKHvubKRCE0v2ONlM4=

Dockerfile

@@ -2,9 +2,9 @@ FROM golang:alpine as builder
 RUN apk --update --no-cache add make git g++
 
 # Build statically linked vDB binary (wonky path because of Dep)
-RUN mkdir -p /go/src/github.com/vulcanize/vulcanizedb
-ADD . /go/src/github.com/vulcanize/vulcanizedb
-WORKDIR /go/src/github.com/vulcanize/vulcanizedb
+RUN mkdir -p /go/src/github.com/vulcanize/ipfs-chain-watcher
+ADD . /go/src/github.com/vulcanize/ipfs-chain-watcher
+WORKDIR /go/src/github.com/vulcanize/ipfs-chain-watcher
 RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' .
 
 # Build migration tool
@@ -14,10 +14,10 @@ RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflag
 
 # Second stage
 FROM alpine
-COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb /app/vulcanizedb
-COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/environments/staging.toml /app/environments/
-COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/startup_script.sh /app/
-COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations/* /app/
+COPY --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/vulcanizedb /app/vulcanizedb
+COPY --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/environments/staging.toml /app/environments/
+COPY --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/dockerfiles/startup_script.sh /app/
+COPY --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/db/migrations/* /app/
 COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose /app/goose
 
 WORKDIR /app

Makefile

@@ -41,22 +41,6 @@ metalint: | $(METALINT)
 lint:
 	$(LINT) $$($(PKGS)) | grep -v -E "exported (function)|(var)|(method)|(type).*should have comment or be unexported"
 
-.PHONY: test
-test: | $(GINKGO) $(LINT)
-	go vet ./...
-	go fmt ./...
-	$(GINKGO) -r --skipPackage=integration_tests,integration
-
-.PHONY: integrationtest
-integrationtest: | $(GINKGO) $(LINT)
-	go vet ./...
-	go fmt ./...
-	$(GINKGO) -r integration_test/
-
-build:
-	go fmt ./...
-	go build
-
 #Database
 HOST_NAME = localhost
 PORT = 5432
@@ -64,6 +48,36 @@ NAME =
 USER = postgres
 CONNECT_STRING=postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable
 
+#Test
+TEST_DB = vulcanize_testing
+TEST_CONNECT_STRING = postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(TEST_DB)?sslmode=disable
+
+.PHONY: test
+test: | $(GINKGO) $(LINT)
+	go vet ./...
+	go fmt ./...
+	dropdb --if-exists $(TEST_DB)
+	createdb $(TEST_DB)
+	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" up
+	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" reset
+	make migrate NAME=$(TEST_DB)
+	$(GINKGO) -r --skipPackage=integration_tests,integration
+
+.PHONY: integrationtest
+integrationtest: | $(GINKGO) $(LINT)
+	go vet ./...
+	go fmt ./...
+	dropdb --if-exists $(TEST_DB)
+	createdb $(TEST_DB)
+	$(GOOSE) -dir db/migrations "$(TEST_CONNECT_STRING)" up
+	$(GOOSE) -dir db/migrations "$(TEST_CONNECT_STRING)" reset
+	make migrate NAME=$(TEST_DB)
+	$(GINKGO) -r integration_test/
+
+build:
+	go fmt ./...
+	go build
+
 # Parameter checks
 ## Check that DB variables are provided
 .PHONY: checkdbvars
@@ -71,7 +85,6 @@ checkdbvars:
 	test -n "$(HOST_NAME)" # $$HOST_NAME
 	test -n "$(PORT)" # $$PORT
 	test -n "$(NAME)" # $$NAME
-	test -n "$(USER)" # $$USER
 	@echo $(CONNECT_STRING)
 
 ## Check that the migration variable (id/timestamp) is provided
@@ -88,41 +101,36 @@ checkmigname:
 ## Rollback the last migration
 .PHONY: rollback
 rollback: $(GOOSE) checkdbvars
-	cd db/migrations;\
-	  $(GOOSE) postgres "$(CONNECT_STRING)" down
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down
 	pg_dump -O -s $(CONNECT_STRING) > db/schema.sql
 
 
 ## Rollbackt to a select migration (id/timestamp)
 .PHONY: rollback_to
 rollback_to: $(GOOSE) checkmigration checkdbvars
-	cd db/migrations;\
-	  $(GOOSE) postgres "$(CONNECT_STRING)" down-to "$(MIGRATION)"
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down-to "$(MIGRATION)"
 
 ## Apply all migrations not already run
 .PHONY: migrate
 migrate: $(GOOSE) checkdbvars
-	cd db/migrations;\
-	  $(GOOSE) postgres "$(CONNECT_STRING)" up
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up
 	pg_dump -O -s $(CONNECT_STRING) > db/schema.sql
 
 ## Create a new migration file
 .PHONY: new_migration
 new_migration: $(GOOSE) checkmigname
-	cd db/migrations;\
-	  $(GOOSE) create $(NAME) sql
+	$(GOOSE) -dir db/migrations create $(NAME) sql
 
 ## Check which migrations are applied at the moment
 .PHONY: migration_status
 migration_status: $(GOOSE) checkdbvars
-	cd db/migrations;\
-	  $(GOOSE) postgres "$(CONNECT_STRING)" status
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" status
 
 # Convert timestamped migrations to versioned (to be run in CI);
 # merge timestamped files to prevent conflict
 .PHONY: version_migrations
 version_migrations:
-	cd db/migrations; $(GOOSE) fix
+	$(GOOSE) -dir db/migrations fix
 
 # Import a psql schema to the database
 .PHONY: import

README.md

@@ -1,73 +1,40 @@
 # Vulcanize DB
 
-[![Build Status](https://travis-ci.org/vulcanize/vulcanizedb.svg?branch=master)](https://travis-ci.org/vulcanize/vulcanizedb)
-[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/vulcanizedb)](https://goreportcard.com/report/github.com/vulcanize/vulcanizedb)
-
-> Vulcanize DB is a set of tools that make it easier for developers to write application-specific indexes and caches for dapps built on Ethereum.
+[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/ipfs-blockchain-watcher)](https://goreportcard.com/report/github.com/vulcanize/ipfs-blockchain-watcher)
 
+> Tool for extracting and indexing blockchain data on PG-IPFS
 
 ## Table of Contents
 1. [Background](#background)
+1. [Architecture](#architecture)
 1. [Install](#install)
 1. [Usage](#usage)
 1. [Contributing](#contributing)
 1. [License](#license)
 
-
 ## Background
-The same data structures and encodings that make Ethereum an effective and trust-less distributed virtual machine
-complicate data accessibility and usability for dApp developers. VulcanizeDB improves Ethereum data accessibility by
-providing a suite of tools to ease the extraction and transformation of data into a more useful state, including
-allowing for exposing aggregate data from a suite of smart contracts.
+ipfs-blockchain-watcher is a collection of interfaces that are used to extract, process, and store in Postgres-IPFS
+all chain data. The raw data indexed by ipfs-blockchain-watcher serves as the basis for more specific watchers and applications.
 
-VulanizeDB includes processes that sync, transform and expose data. Syncing involves
-querying an Ethereum node and then persisting core data into a Postgres database. Transforming focuses on using previously synced data to
-query for and transform log event and storage data for specifically configured smart contract addresses. Exposing data is a matter of getting
-data from VulcanizeDB's underlying Postgres database and making it accessible.
+Currently the service supports complete processing of all Bitcoin and Ethereum data.
 
-![VulcanizeDB Overview Diagram](documentation/diagrams/vdb-overview.png)
+## Architecture
+More details on the design of ipfs-blockchain-watcher can be found in [here](./documentation/architecture.md)
 
 ## Install
+1. [Postgres](#postgres)
+1. [Goose](#goose)
+1. [IPFS](#ipfs)
+1. [Blockchain](#blockchain)
+1. [Watcher](#watcher)
 
-1. [Dependencies](#dependencies)
-1. [Building the project](#building-the-project)
-1. [Setting up the database](#setting-up-the-database)
-1. [Configuring a synced Ethereum node](#configuring-a-synced-ethereum-node)
-
-### Dependencies
- - Go 1.12+
- - Postgres 11.2
- - Ethereum Node
-   - [Go Ethereum](https://ethereum.github.io/go-ethereum/downloads/) (1.8.23+)
-   - [Parity 1.8.11+](https://github.com/paritytech/parity/releases)
-
-### Building the project
-Download the codebase to your local `GOPATH` via:
-
-`go get github.com/vulcanize/vulcanizedb`
-
-Move to the project directory:
-
-`cd $GOPATH/src/github.com/vulcanize/vulcanizedb`
-
-Be sure you have enabled Go Modules (`export GO111MODULE=on`), and build the executable with:
-
-`make build`
-
-If you need to use a different dependency than what is currently defined in `go.mod`, it may helpful to look into [the replace directive](https://github.com/golang/go/wiki/Modules#when-should-i-use-the-replace-directive).
-This instruction enables you to point at a fork or the local filesystem for dependency resolution.
-
-If you are running into issues at this stage, ensure that `GOPATH` is defined in your shell.
-If necessary, `GOPATH` can be set in `~/.bashrc` or `~/.bash_profile`, depending upon your system.
-It can be additionally helpful to add `$GOPATH/bin` to your shell's `$PATH`.
-
-### Setting up the database
-1. Install Postgres
+### Postgres
+1. [Install Postgres](https://wiki.postgresql.org/wiki/Detailed_installation_guides)
 1. Create a superuser for yourself and make sure `psql --list` works without prompting for a password.
 1. `createdb vulcanize_public`
-1. `cd $GOPATH/src/github.com/vulcanize/vulcanizedb`
+1. `cd $GOPATH/src/github.com/vulcanize/ipfs-blockchain-watcher`
 1.  Run the migrations: `make migrate HOST_NAME=localhost NAME=vulcanize_public PORT=5432`
-    - There is an optional var `USER=username` if the database user is not the default user `postgres`
+    - There are optional vars `USER=username` and `PASS=password` if the database user is not the default user `postgres` and/or a password is present
     - To rollback a single step: `make rollback NAME=vulcanize_public`
     - To rollback to a certain migration: `make rollback_to MIGRATION=n NAME=vulcanize_public`
     - To see status of migrations: `make migration_status NAME=vulcanize_public`
@@ -79,76 +46,207 @@ localhost. To allow access on Ubuntu, set localhost connections via hostname, ip
 
 (It should be noted that trusted auth should only be enabled on systems without sensitive data in them: development and local test databases)
 
-### Configuring a synced Ethereum node
-- To use a local Ethereum node, copy `environments/public.toml.example` to
-  `environments/public.toml` and update the `ipcPath` and `levelDbPath`.
-  - `ipcPath` should match the local node's IPC filepath:
-      - For Geth:
-        - The IPC file is called `geth.ipc`.
-        - The geth IPC file path is printed to the console when you start geth.
-        - The default location is:
-          - Mac: `<full home path>/Library/Ethereum/geth.ipc`
-          - Linux: `<full home path>/ethereum/geth.ipc`
-        - Note: the geth.ipc file may not exist until you've started the geth process
+### Goose
+We use [goose](https://github.com/pressly/goose) as our migration management tool. While it is not necessary to use `goose` for manual setup, it
+is required for running the automated tests.
 
-      - For Parity:
-        - The IPC file is called `jsonrpc.ipc`.
-        - The default location is:
-          - Mac: `<full home path>/Library/Application\ Support/io.parity.ethereum/`
-          - Linux: `<full home path>/local/share/io.parity.ethereum/`
+### IPFS
+We use IPFS to store IPLD objects for each type of data we extract from on chain.
 
-  - `levelDbPath` should match Geth's chaindata directory path.
-      - The geth LevelDB chaindata path is printed to the console when you start geth.
-      - The default location is:
-          - Mac: `<full home path>/Library/Ethereum/geth/chaindata`
-          - Linux: `<full home path>/ethereum/geth/chaindata`
-      - `levelDbPath` is irrelevant (and `coldImport` is currently unavailable) if only running parity.
+To start, download and install [IPFS](https://github.com/vulcanize/go-ipfs):
 
+`go get github.com/ipfs/go-ipfs`
+
+`cd $GOPATH/src/github.com/ipfs/go-ipfs`
+
+`make install`
+
+If we want to use Postgres as our backing datastore, we need to use the vulcanize fork of go-ipfs.
+
+Start by adding the fork and switching over to it:
+
+`git remote add vulcanize https://github.com/vulcanize/go-ipfs.git`
+
+`git fetch vulcanize`
+
+`git checkout -b postgres_update vulcanize/postgres_update`
+
+Now install this fork of ipfs, first be sure to remove any previous installation:
+
+`make install`
+
+Check that is installed properly by running:
+
+`ipfs`
+
+You should see the CLI info/help output.
+
+And now we initialize with the `postgresds` profile.
+If ipfs was previously initialized we will need to remove the old profile first.
+We also need to provide env variables for the postgres connection:
+
+We can either set these manually, e.g.
+```bash
+export IPFS_PGHOST=
+export IPFS_PGUSER=
+export IPFS_PGDATABASE=
+export IPFS_PGPORT=
+export IPFS_PGPASSWORD=
+```
+
+And then run the ipfs command:
+
+`ipfs init --profile=postgresds`
+
+Or we can use the pre-made script at `GOPATH/src/github.com/ipfs/go-ipfs/misc/utility/ipfs_postgres.sh`
+which has usage:
+
+`./ipfs_postgres.sh <IPFS_PGHOST> <IPFS_PGPORT> <IPFS_PGUSER> <IPFS_PGDATABASE>"`
+
+and will ask us to enter the password, avoiding storing it to an ENV variable.
+
+Once we have initialized ipfs, that is all we need to do with it- we do not need to run a daemon during the subsequent processes.
+
+### Blockchain
+This section describes how to setup an Ethereum or Bitcoin node to serve as a data source for ipfs-blockchain-watcher
+
+#### Ethereum
+For Ethereum, we currently *require* [a special fork of go-ethereum](https://github.com/vulcanize/go-ethereum/tree/statediff_at_anyblock-1.9.11). This can be setup as follows.
+Skip this steps if you already have access to a node that displays the statediffing endpoints.
+
+Begin by downloading geth and switching to the vulcanize/rpc_statediffing branch:
+
+`go get github.com/ethereum/go-ethereum`
+
+`cd $GOPATH/src/github.com/ethereum/go-ethereum`
+
+`git remote add vulcanize https://github.com/vulcanize/go-ethereum.git`
+
+`git fetch vulcanize`
+
+`git checkout -b statediffing vulcanize/statediff_at_anyblock-1.9.11`
+
+Now, install this fork of geth (make sure any old versions have been uninstalled/binaries removed first):
+
+`make geth`
+
+And run the output binary with statediffing turned on:
+
+`cd $GOPATH/src/github.com/ethereum/go-ethereum/build/bin`
+
+`./geth --statediff --statediff.streamblock --ws --syncmode=full`
+
+Note: if you wish to access historical data (perform `backFill`) then the node will need to operate as an archival node (`--gcmode=archive`)
+
+Note: other CLI options- statediff specific ones included- can be explored with `./geth help`
+
+The output from geth should mention that it is `Starting statediff service` and block synchronization should begin shortly thereafter.
+Note that until it receives a subscriber, the statediffing process does nothing but wait for one. Once a subscription is received, this
+will be indicated in the output and node will begin processing and sending statediffs.
+
+Also in the output will be the endpoints that we will use to interface with the node.
+The default ws url is "127.0.0.1:8546" and the default http url is "127.0.0.1:8545".
+These values will be used as the `ethereum.wsPath` and `ethereum.httpPath` in the config, respectively.
+
+#### Bitcoin
+For Bitcoin, ipfs-blockchain-watcher is able to operate entirely through the universally exposed JSON-RPC interfaces.
+This means we can use any of the standard full nodes (e.g. bitcoind, btcd) as our data source.
+
+Point at a remote node or set one up locally using the instructions for [bitcoind](https://github.com/bitcoin/bitcoin) and [btcd](https://github.com/btcsuite/btcd).
+
+The default http url is "127.0.0.1:8332". We will use the http endpoint as both the `bitcoin.wsPath` and `bitcoin.httpPath`
+(bitcoind does not support websocket endpoints, we are currently using a "subscription" wrapper around the http endpoints)
+
+### Watcher
+Finally, we can setup the watcher process itself.
+
+Start by downloading vulcanizedb and moving into the repo:
+
+`go get github.com/vulcanize/ipfs-chain-watcher`
+
+`cd $GOPATH/src/github.com/vulcanize/ipfs-chain-watcher`
+
+Then, build the binary:
+
+`make build`
 
 ## Usage
-As mentioned above, VulcanizeDB's processes can be split into three categories: syncing, transforming and exposing data.
+After building the binary, run as
 
-### Data syncing
-To provide data for transformations, raw Ethereum data must first be synced into VulcanizeDB.
-This is accomplished through the use of the `headerSync`, `fullSync`, or `coldImport` commands.
-These commands are described in detail [here](documentation/data-syncing.md).
+`./ipfs-blockchain-watcher watch --config=<config_file.toml`
 
-### Data transformation
-Data transformation uses the raw data that has been synced into Postgres to filter out and apply transformations to
-specific data of interest. Since there are different types of data that may be useful for observing smart contracts, it
-follows that there are different ways to transform this data. We've started by categorizing this into Generic and
-Custom transformers:
+### Configuration
 
-- Generic Contract Transformer: Generic contract transformation can be done using a built-in command,
-`contractWatcher`, which transforms contract events provided the contract's ABI is available. It also
-provides some state variable coverage by automating polling of public methods, with some restrictions.
-`contractWatcher` is described further [here](documentation/generic-transformer.md).
+Below is the set of universal config parameters for the ipfs-blockchain-watcher command, in .toml form, with the respective environmental variables commented to the side.
+This set of parameters needs to be set no matter the chain type.
 
-- Custom Transformers: In many cases custom transformers will need to be written to provide
-more comprehensive coverage of contract data. In this case we have provided the `compose`, `execute`, and
-`composeAndExecute` commands for running custom transformers from external repositories. Documentation on how to write,
-build and run custom transformers as Go plugins can be found
-[here](documentation/custom-transformers.md).
+```toml
+[database]
+    name     = "vulcanize_public" # $DATABASE_NAME
+    hostname = "localhost" # $DATABASE_HOSTNAME
+    port     = 5432 # $DATABASE_PORT
+    user     = "vdbm" # $DATABASE_USER
+    password = "" # $DATABASE_PASSWORD
+
+[ipfs]
+    path = "~/.ipfs" # $IPFS_PATH
+    mode = "postgres" # $IPFS_MODE
+
+[superNode]
+    chain = "bitcoin" # $SUPERNODE_CHAIN
+    server = true # $SUPERNODE_SERVER
+    ipcPath = "~/.vulcanize/vulcanize.ipc" # $SUPERNODE_IPC_PATH
+    wsPath = "127.0.0.1:8082" # $SUPERNODE_WS_PATH
+    httpPath = "127.0.0.1:8083" # $SUPERNODE_HTTP_PATH
+    sync = true # $SUPERNODE_SYNC
+    workers = 1 # $SUPERNODE_WORKERS
+    backFill = true # $SUPERNODE_BACKFILL
+    frequency = 45 # $SUPERNODE_FREQUENCY
+    batchSize = 1 # $SUPERNODE_BATCH_SIZE
+    batchNumber = 50 # $SUPERNODE_BATCH_NUMBER
+    timeout = 300 # $HTTP_TIMEOUT
+    validationLevel = 1 # $SUPERNODE_VALIDATION_LEVEL
+```
+
+Additional parameters need to be set depending on the specific chain.
+
+For Bitcoin:
+
+```toml
+[bitcoin]
+    wsPath  = "127.0.0.1:8332" # $BTC_WS_PATH
+    httpPath = "127.0.0.1:8332" # $BTC_HTTP_PATH
+    pass = "password" # $BTC_NODE_PASSWORD
+    user = "username" # $BTC_NODE_USER
+    nodeID = "ocd0" # $BTC_NODE_ID
+    clientName = "Omnicore" # $BTC_CLIENT_NAME
+    genesisBlock = "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f" # $BTC_GENESIS_BLOCK
+    networkID = "0xD9B4BEF9" # $BTC_NETWORK_ID
+```
+
+For Ethereum:
+
+```toml
+[ethereum]
+    wsPath  = "127.0.0.1:8546" # $ETH_WS_PATH
+    httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH
+    nodeID = "arch1" # $ETH_NODE_ID
+    clientName = "Geth" # $ETH_CLIENT_NAME
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK
+    networkID = "1" # $ETH_NETWORK_ID
+```
 
 ### Exposing the data
-[Postgraphile](https://www.graphile.org/postgraphile/) is used to expose GraphQL endpoints for our database schemas, this is described in detail [here](documentation/postgraphile.md).
-
-
-### Tests
-- Replace the empty `ipcPath` in the `environments/infura.toml` with a path to a full node's eth_jsonrpc endpoint (e.g. local geth node ipc path or infura url)
-    - Note: integration tests require configuration with an archival node
-- `createdb vulcanize_private` will create the test db
-- `make migrate NAME=vulcanize_private` will run the db migrations
-- `make test` will run the unit tests and skip the integration tests
-- `make integrationtest` will run just the integration tests
+We can expose a number of different APIs for remote access to ipfs-blockchain-watcher data, these are dicussed in more detail [here](./documentation/apis.md)
 
+### Testing
+`make test` will run the unit tests
+`make test` setups a clean `vulcanize_testing` db
 
 ## Contributing
 Contributions are welcome!
 
 VulcanizeDB follows the [Contributor Covenant Code of Conduct](https://www.contributor-covenant.org/version/1/4/code-of-conduct).
 
-For more information on contributing, please see [here](documentation/contributing.md).
-
 ## License
 [AGPL-3.0](LICENSE) © Vulcanize Inc

cmd/coldImport.go

@@ -1,97 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/vulcanize/vulcanizedb/pkg/crypto"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/ethereum"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/fs"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/cold_import"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/converters/cold_db"
-	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
-	"github.com/vulcanize/vulcanizedb/utils"
-)
-
-var coldImportCmd = &cobra.Command{
-	Use:   "coldImport",
-	Short: "Sync vulcanize from a cold instance of LevelDB.",
-	Long: `Populate core vulcanize db data directly out of LevelDB, rather than over rpc calls. For example:
-
-./vulcanizedb coldImport -s 0 -e 5000000
-
-Geth must be synced over all of the desired blocks and must not be running in order to execute this command.`,
-	Run: func(cmd *cobra.Command, args []string) {
-		SubCommand = cmd.CalledAs()
-		LogWithCommand = *log.WithField("SubCommand", SubCommand)
-		coldImport()
-	},
-}
-
-func init() {
-	rootCmd.AddCommand(coldImportCmd)
-	coldImportCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "BlockNumber for first block to cold import.")
-	coldImportCmd.Flags().Int64VarP(&endingBlockNumber, "ending-block-number", "e", 5500000, "BlockNumber for last block to cold import.")
-	coldImportCmd.Flags().BoolVarP(&syncAll, "all", "a", false, "Option to sync all missing blocks.")
-}
-
-func coldImport() {
-	// init eth db
-	ethDBConfig := ethereum.CreateDatabaseConfig(ethereum.Level, levelDbPath)
-	ethDB, err := ethereum.CreateDatabase(ethDBConfig)
-	if err != nil {
-		LogWithCommand.Fatal("Error connecting to ethereum db: ", err)
-	}
-	mostRecentBlockNumberInDb := ethDB.GetHeadBlockNumber()
-	if syncAll {
-		startingBlockNumber = 0
-		endingBlockNumber = mostRecentBlockNumberInDb
-	}
-	if endingBlockNumber < startingBlockNumber {
-		LogWithCommand.Fatal("Ending block number must be greater than starting block number for cold import.")
-	}
-	if endingBlockNumber > mostRecentBlockNumberInDb {
-		LogWithCommand.Fatal("Ending block number is greater than most recent block in db: ", mostRecentBlockNumberInDb)
-	}
-
-	// init pg db
-	genesisBlock := ethDB.GetBlockHash(0)
-	reader := fs.FsReader{}
-	parser := crypto.EthPublicKeyParser{}
-	nodeBuilder := cold_import.NewColdImportNodeBuilder(reader, parser)
-	coldNode, err := nodeBuilder.GetNode(genesisBlock, levelDbPath)
-	if err != nil {
-		LogWithCommand.Fatal("Error getting node: ", err)
-	}
-	pgDB := utils.LoadPostgres(databaseConfig, coldNode)
-
-	// init cold importer deps
-	blockRepository := repositories.NewBlockRepository(&pgDB)
-	receiptRepository := repositories.FullSyncReceiptRepository{DB: &pgDB}
-	transactionConverter := cold_db.NewColdDbTransactionConverter()
-	blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
-
-	// init and execute cold importer
-	coldImporter := cold_import.NewColdImporter(ethDB, blockRepository, receiptRepository, blockConverter)
-	err = coldImporter.Execute(startingBlockNumber, endingBlockNumber, coldNode.ID)
-	if err != nil {
-		LogWithCommand.Fatal("Error executing cold import: ", err)
-	}
-}

cmd/compose.go

@@ -1,192 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"errors"
-	"fmt"
-	"strconv"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
-)
-
-// composeCmd represents the compose command
-var composeCmd = &cobra.Command{
-	Use:   "compose",
-	Short: "Composes transformer initializer plugin",
-	Long: `This command needs a config .toml file of form:
-
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    user     = "vulcanize"
-    password = "vulcanize"
-    port     = 5432
-
-[client]
-    ipcPath  = "/Users/user/Library/Ethereum/geth.ipc"
-
-[exporter]
-    home     = "github.com/vulcanize/vulcanizedb"
-    name     = "exampleTransformerExporter"
-    save     = false
-    transformerNames = [
-        "transformer1",
-        "transformer2",
-        "transformer3",
-        "transformer4",
-    ]
-    [exporter.transformer1]
-        path = "path/to/transformer1"
-        type = "eth_event"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "0"
-    [exporter.transformer2]
-        path = "path/to/transformer2"
-        type = "eth_contract"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "0"
-    [exporter.transformer3]
-        path = "path/to/transformer3"
-        type = "eth_event"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "0"
-    [exporter.transformer4]
-        path = "path/to/transformer4"
-        type = "eth_storage"
-        repository = "github.com/account2/repo2"
-        migrations = "to/db/migrations"
-        rank = "1"
-
-
-Note: If any of the plugin transformer need additional
-configuration variables include them in the .toml file as well
-
-This information is used to write and build a go plugin with a transformer 
-set composed from the transformer imports specified in the config file
-This plugin is loaded and the set of transformer initializers is exported
-from it and loaded into and executed over by the appropriate watcher. 
-
-The type of watcher that the transformer works with is specified using the 
-type variable for each transformer in the config. Currently there are watchers 
-of event data from an eth node (eth_event) and storage data from an eth node 
-(eth_storage), and a more generic interface for accepting contract_watcher pkg
-based transformers which can perform both event watching and public method 
-polling (eth_contract).
-
-Transformers of different types can be ran together in the same command using a 
-single config file or in separate command instances using different config files
-
-Specify config location when executing the command:
-./vulcanizedb compose --config=./environments/config_name.toml`,
-	Run: func(cmd *cobra.Command, args []string) {
-		SubCommand = cmd.CalledAs()
-		LogWithCommand = *log.WithField("SubCommand", SubCommand)
-		compose()
-	},
-}
-
-func compose() {
-	// Build plugin generator config
-	prepConfig()
-
-	// Generate code to build the plugin according to the config file
-	LogWithCommand.Info("generating plugin")
-	generator, err := p2.NewGenerator(genConfig, databaseConfig)
-	if err != nil {
-		LogWithCommand.Debug("initializing plugin generator failed")
-		LogWithCommand.Fatal(err)
-	}
-	err = generator.GenerateExporterPlugin()
-	if err != nil {
-		LogWithCommand.Debug("generating plugin failed")
-		LogWithCommand.Fatal(err)
-	}
-	// TODO: Embed versioning info in the .so files so we know which version of vulcanizedb to run them with
-	_, pluginPath, err := genConfig.GetPluginPaths()
-	if err != nil {
-		LogWithCommand.Debug("getting plugin path failed")
-		LogWithCommand.Fatal(err)
-	}
-	fmt.Printf("Composed plugin %s", pluginPath)
-	LogWithCommand.Info("plugin .so file output to ", pluginPath)
-}
-
-func init() {
-	rootCmd.AddCommand(composeCmd)
-}
-
-func prepConfig() {
-	LogWithCommand.Info("configuring plugin")
-	names := viper.GetStringSlice("exporter.transformerNames")
-	transformers := make(map[string]config.Transformer)
-	for _, name := range names {
-		transformer := viper.GetStringMapString("exporter." + name)
-		p, pOK := transformer["path"]
-		if !pOK || p == "" {
-			LogWithCommand.Fatal(name, " transformer config is missing `path` value")
-		}
-		r, rOK := transformer["repository"]
-		if !rOK || r == "" {
-			LogWithCommand.Fatal(name, " transformer config is missing `repository` value")
-		}
-		m, mOK := transformer["migrations"]
-		if !mOK || m == "" {
-			LogWithCommand.Fatal(name, " transformer config is missing `migrations` value")
-		}
-		mr, mrOK := transformer["rank"]
-		if !mrOK || mr == "" {
-			LogWithCommand.Fatal(name, " transformer config is missing `rank` value")
-		}
-		rank, err := strconv.ParseUint(mr, 10, 64)
-		if err != nil {
-			LogWithCommand.Fatal(name, " migration `rank` can't be converted to an unsigned integer")
-		}
-		t, tOK := transformer["type"]
-		if !tOK {
-			LogWithCommand.Fatal(name, " transformer config is missing `type` value")
-		}
-		transformerType := config.GetTransformerType(t)
-		if transformerType == config.UnknownTransformerType {
-			LogWithCommand.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`))
-		}
-
-		transformers[name] = config.Transformer{
-			Path:           p,
-			Type:           transformerType,
-			RepositoryPath: r,
-			MigrationPath:  m,
-			MigrationRank:  rank,
-		}
-	}
-
-	genConfig = config.Plugin{
-		Transformers: transformers,
-		FilePath:     "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins",
-		FileName:     viper.GetString("exporter.name"),
-		Save:         viper.GetBool("exporter.save"),
-		Home:         viper.GetString("exporter.home"),
-	}
-}

cmd/composeAndExecute.go

@@ -1,203 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"os"
-	"plugin"
-	syn "sync"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
-	"github.com/vulcanize/vulcanizedb/pkg/fs"
-	p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
-	"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
-	"github.com/vulcanize/vulcanizedb/utils"
-)
-
-// composeAndExecuteCmd represents the composeAndExecute command
-var composeAndExecuteCmd = &cobra.Command{
-	Use:   "composeAndExecute",
-	Short: "Composes, loads, and executes transformer initializer plugin",
-	Long: `This command needs a config .toml file of form:
-
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    user     = "vulcanize"
-    password = "vulcanize"
-    port     = 5432
-
-[client]
-    ipcPath  = "/Users/user/Library/Ethereum/geth.ipc"
-
-[exporter]
-    home     = "github.com/vulcanize/vulcanizedb"
-    name     = "exampleTransformerExporter"
-    save     = false
-    transformerNames = [
-        "transformer1",
-        "transformer2",
-        "transformer3",
-        "transformer4",
-    ]
-    [exporter.transformer1]
-        path = "path/to/transformer1"
-        type = "eth_event"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "0"
-    [exporter.transformer2]
-        path = "path/to/transformer2"
-        type = "eth_contract"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "2"
-    [exporter.transformer3]
-        path = "path/to/transformer3"
-        type = "eth_event"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "0"
-    [exporter.transformer4]
-        path = "path/to/transformer4"
-        type = "eth_storage"
-        repository = "github.com/account2/repo2"
-        migrations = "to/db/migrations"
-        rank = "1"
-
-
-Note: If any of the plugin transformer need additional
-configuration variables include them in the .toml file as well
-
-This information is used to write and build a go plugin with a transformer 
-set composed from the transformer imports specified in the config file
-This plugin is loaded and the set of transformer initializers is exported
-from it and loaded into and executed over by the appropriate watcher. 
-
-The type of watcher that the transformer works with is specified using the 
-type variable for each transformer in the config. Currently there are watchers 
-of event data from an eth node (eth_event) and storage data from an eth node 
-(eth_storage), and a more generic interface for accepting contract_watcher pkg
-based transformers which can perform both event watching and public method 
-polling (eth_contract).
-
-Transformers of different types can be ran together in the same command using a 
-single config file or in separate command instances using different config files
-
-Specify config location when executing the command:
-./vulcanizedb composeAndExecute --config=./environments/config_name.toml`,
-	Run: func(cmd *cobra.Command, args []string) {
-		SubCommand = cmd.CalledAs()
-		LogWithCommand = *log.WithField("SubCommand", SubCommand)
-		composeAndExecute()
-	},
-}
-
-func composeAndExecute() {
-	// Build plugin generator config
-	prepConfig()
-
-	// Generate code to build the plugin according to the config file
-	LogWithCommand.Info("generating plugin")
-	generator, err := p2.NewGenerator(genConfig, databaseConfig)
-	if err != nil {
-		LogWithCommand.Fatal(err)
-	}
-	err = generator.GenerateExporterPlugin()
-	if err != nil {
-		LogWithCommand.Debug("generating plugin failed")
-		LogWithCommand.Fatal(err)
-	}
-
-	// Get the plugin path and load the plugin
-	_, pluginPath, err := genConfig.GetPluginPaths()
-	if err != nil {
-		LogWithCommand.Fatal(err)
-	}
-	if !genConfig.Save {
-		defer helpers.ClearFiles(pluginPath)
-	}
-	LogWithCommand.Info("linking plugin ", pluginPath)
-	plug, err := plugin.Open(pluginPath)
-	if err != nil {
-		LogWithCommand.Debug("linking plugin failed")
-		LogWithCommand.Fatal(err)
-	}
-
-	// Load the `Exporter` symbol from the plugin
-	LogWithCommand.Info("loading transformers from plugin")
-	symExporter, err := plug.Lookup("Exporter")
-	if err != nil {
-		LogWithCommand.Debug("loading Exporter symbol failed")
-		LogWithCommand.Fatal(err)
-	}
-
-	// Assert that the symbol is of type Exporter
-	exporter, ok := symExporter.(Exporter)
-	if !ok {
-		LogWithCommand.Debug("plugged-in symbol not of type Exporter")
-		os.Exit(1)
-	}
-
-	// Use the Exporters export method to load the EventTransformerInitializer, StorageTransformerInitializer, and ContractTransformerInitializer sets
-	ethEventInitializers, ethStorageInitializers, ethContractInitializers := exporter.Export()
-
-	// Setup bc and db objects
-	blockChain := getBlockChain()
-	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
-
-	// Execute over transformer sets returned by the exporter
-	// Use WaitGroup to wait on both goroutines
-	var wg syn.WaitGroup
-	if len(ethEventInitializers) > 0 {
-		ew := watcher.NewEventWatcher(&db, blockChain)
-		err := ew.AddTransformers(ethEventInitializers)
-		if err != nil {
-			LogWithCommand.Fatalf("failed to add event transformer initializers to watcher: %s", err.Error())
-		}
-		wg.Add(1)
-		go watchEthEvents(&ew, &wg)
-	}
-
-	if len(ethStorageInitializers) > 0 {
-		tailer := fs.FileTailer{Path: storageDiffsPath}
-		storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
-		sw := watcher.NewStorageWatcher(storageFetcher, &db)
-		sw.AddTransformers(ethStorageInitializers)
-		wg.Add(1)
-		go watchEthStorage(&sw, &wg)
-	}
-
-	if len(ethContractInitializers) > 0 {
-		gw := watcher.NewContractWatcher(&db, blockChain)
-		gw.AddTransformers(ethContractInitializers)
-		wg.Add(1)
-		go watchEthContract(&gw, &wg)
-	}
-	wg.Wait()
-}
-
-func init() {
-	rootCmd.AddCommand(composeAndExecuteCmd)
-	composeAndExecuteCmd.Flags().BoolVarP(&recheckHeadersArg, "recheck-headers", "r", false, "whether to re-check headers for watched events")
-	composeAndExecuteCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "interval duration for rechecking queued storage diffs (ex: 5m30s)")
-}

cmd/contractWatcher.go

@@ -1,127 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"fmt"
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	st "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	ft "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer"
-	ht "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/transformer"
-	"github.com/vulcanize/vulcanizedb/utils"
-)
-
-// contractWatcherCmd represents the contractWatcher command
-var contractWatcherCmd = &cobra.Command{
-	Use:   "contractWatcher",
-	Short: "Watches events at the provided contract address using fully synced vDB",
-	Long: `Uses input contract address and event filters to watch events
-
-Expects an ethereum node to be running
-Expects an archival node synced into vulcanizeDB
-Requires a .toml config file:
-
-  [database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    port     = 5432
-
-  [client]
-    ipcPath  = "/Users/user/Library/Ethereum/geth.ipc"
-
-  [contract]
-    network  = ""
-    addresses  = [
-        "contractAddress1",
-        "contractAddress2"
-    ]
-    [contract.contractAddress1]
-        abi    = 'ABI for contract 1'
-        startingBlock = 982463
-    [contract.contractAddress2]
-        abi    = 'ABI for contract 2'
-        events = [
-            "event1",
-            "event2"
-        ]
-		eventArgs = [
-			"arg1",
-			"arg2"
-		]
-        methods = [
-            "method1",
-			"method2"
-        ]
-		methodArgs = [
-			"arg1",
-			"arg2"
-		]
-        startingBlock = 4448566
-        piping = true
-`,
-	Run: func(cmd *cobra.Command, args []string) {
-		SubCommand = cmd.CalledAs()
-		LogWithCommand = *log.WithField("SubCommand", SubCommand)
-		contractWatcher()
-	},
-}
-
-var (
-	mode string
-)
-
-func contractWatcher() {
-	ticker := time.NewTicker(5 * time.Second)
-	defer ticker.Stop()
-
-	blockChain := getBlockChain()
-	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
-
-	var t st.ContractTransformer
-	con := config.ContractConfig{}
-	con.PrepConfig()
-	switch mode {
-	case "header":
-		t = ht.NewTransformer(con, blockChain, &db)
-	case "full":
-		t = ft.NewTransformer(con, blockChain, &db)
-	default:
-		LogWithCommand.Fatal("Invalid mode")
-	}
-
-	err := t.Init()
-	if err != nil {
-		LogWithCommand.Fatal(fmt.Sprintf("Failed to initialize transformer, err: %v ", err))
-	}
-
-	for range ticker.C {
-		err = t.Execute()
-		if err != nil {
-			LogWithCommand.Error("Execution error for transformer: ", t.GetConfig().Name, err)
-		}
-	}
-}
-
-func init() {
-	rootCmd.AddCommand(contractWatcherCmd)
-	contractWatcherCmd.Flags().StringVarP(&mode, "mode", "o", "header", "'header' or 'full' mode to work with either header synced or fully synced vDB (default is header)")
-}

cmd/execute.go

@@ -1,191 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"fmt"
-	"plugin"
-	syn "sync"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
-	storageUtils "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
-	"github.com/vulcanize/vulcanizedb/pkg/fs"
-	"github.com/vulcanize/vulcanizedb/utils"
-)
-
-// executeCmd represents the execute command
-var executeCmd = &cobra.Command{
-	Use:   "execute",
-	Short: "executes a precomposed transformer initializer plugin",
-	Long: `This command needs a config .toml file of form:
-
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    user     = "vulcanize"
-    password = "vulcanize"
-    port     = 5432
-
-[client]
-    ipcPath  = "/Users/user/Library/Ethereum/geth.ipc"
-
-[exporter]
-    name     = "exampleTransformerExporter"
-
-Note: If any of the plugin transformer need additional
-configuration variables include them in the .toml file as well
-
-The exporter.name is the name (without extension) of the plugin to be loaded.
-The plugin file needs to be located in the /plugins directory and this command assumes 
-the db migrations remain from when the plugin was composed. Additionally, the plugin 
-must have been composed by the same version of vulcanizedb or else it will not be compatible.
-
-Specify config location when executing the command:
-./vulcanizedb execute --config=./environments/config_name.toml`,
-	Run: func(cmd *cobra.Command, args []string) {
-		SubCommand = cmd.CalledAs()
-		LogWithCommand = *log.WithField("SubCommand", SubCommand)
-		execute()
-	},
-}
-
-func execute() {
-	// Build plugin generator config
-	prepConfig()
-
-	// Get the plugin path and load the plugin
-	_, pluginPath, err := genConfig.GetPluginPaths()
-	if err != nil {
-		LogWithCommand.Fatal(err)
-	}
-
-	fmt.Printf("Executing plugin %s", pluginPath)
-	LogWithCommand.Info("linking plugin ", pluginPath)
-	plug, err := plugin.Open(pluginPath)
-	if err != nil {
-		LogWithCommand.Warn("linking plugin failed")
-		LogWithCommand.Fatal(err)
-	}
-
-	// Load the `Exporter` symbol from the plugin
-	LogWithCommand.Info("loading transformers from plugin")
-	symExporter, err := plug.Lookup("Exporter")
-	if err != nil {
-		LogWithCommand.Warn("loading Exporter symbol failed")
-		LogWithCommand.Fatal(err)
-	}
-
-	// Assert that the symbol is of type Exporter
-	exporter, ok := symExporter.(Exporter)
-	if !ok {
-		LogWithCommand.Fatal("plugged-in symbol not of type Exporter")
-	}
-
-	// Use the Exporters export method to load the EventTransformerInitializer, StorageTransformerInitializer, and ContractTransformerInitializer sets
-	ethEventInitializers, ethStorageInitializers, ethContractInitializers := exporter.Export()
-
-	// Setup bc and db objects
-	blockChain := getBlockChain()
-	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
-
-	// Execute over transformer sets returned by the exporter
-	// Use WaitGroup to wait on both goroutines
-	var wg syn.WaitGroup
-	if len(ethEventInitializers) > 0 {
-		ew := watcher.NewEventWatcher(&db, blockChain)
-		err = ew.AddTransformers(ethEventInitializers)
-		if err != nil {
-			LogWithCommand.Fatalf("failed to add event transformer initializers to watcher: %s", err.Error())
-		}
-		wg.Add(1)
-		go watchEthEvents(&ew, &wg)
-	}
-
-	if len(ethStorageInitializers) > 0 {
-		tailer := fs.FileTailer{Path: storageDiffsPath}
-		storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
-		sw := watcher.NewStorageWatcher(storageFetcher, &db)
-		sw.AddTransformers(ethStorageInitializers)
-		wg.Add(1)
-		go watchEthStorage(&sw, &wg)
-	}
-
-	if len(ethContractInitializers) > 0 {
-		gw := watcher.NewContractWatcher(&db, blockChain)
-		gw.AddTransformers(ethContractInitializers)
-		wg.Add(1)
-		go watchEthContract(&gw, &wg)
-	}
-	wg.Wait()
-}
-
-func init() {
-	rootCmd.AddCommand(executeCmd)
-	executeCmd.Flags().BoolVarP(&recheckHeadersArg, "recheck-headers", "r", false, "whether to re-check headers for watched events")
-	executeCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "interval duration for rechecking queued storage diffs (ex: 5m30s)")
-}
-
-type Exporter interface {
-	Export() ([]transformer.EventTransformerInitializer, []transformer.StorageTransformerInitializer, []transformer.ContractTransformerInitializer)
-}
-
-func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
-	defer wg.Done()
-	// Execute over the EventTransformerInitializer set using the watcher
-	LogWithCommand.Info("executing event transformers")
-	var recheck constants.TransformerExecution
-	if recheckHeadersArg {
-		recheck = constants.HeaderRecheck
-	} else {
-		recheck = constants.HeaderUnchecked
-	}
-	err := w.Execute(recheck)
-	if err != nil {
-		LogWithCommand.Fatalf("error executing event watcher: %s", err.Error())
-	}
-}
-
-func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
-	defer wg.Done()
-	// Execute over the StorageTransformerInitializer set using the storage watcher
-	LogWithCommand.Info("executing storage transformers")
-	ticker := time.NewTicker(pollingInterval)
-	defer ticker.Stop()
-	for range ticker.C {
-		errs := make(chan error)
-		rows := make(chan storageUtils.StorageDiffRow)
-		w.Execute(rows, errs, queueRecheckInterval)
-	}
-}
-
-func watchEthContract(w *watcher.ContractWatcher, wg *syn.WaitGroup) {
-	defer wg.Done()
-	// Execute over the ContractTransformerInitializer set using the contract watcher
-	LogWithCommand.Info("executing contract_watcher transformers")
-	ticker := time.NewTicker(pollingInterval)
-	defer ticker.Stop()
-	for range ticker.C {
-		w.Execute()
-	}
-}

cmd/fullSync.go

@@ -1,105 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"time"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/history"
-	"github.com/vulcanize/vulcanizedb/utils"
-)
-
-// fullSyncCmd represents the fullSync command
-var fullSyncCmd = &cobra.Command{
-	Use:   "fullSync",
-	Short: "Syncs VulcanizeDB with local ethereum node",
-	Long: `Syncs VulcanizeDB with local ethereum node. Populates
-Postgres with blocks, transactions, receipts, and logs.
-
-./vulcanizedb fullSync --starting-block-number 0 --config public.toml
-
-Expects ethereum node to be running and requires a .toml config:
-
-  [database]
-  name = "vulcanize_public"
-  hostname = "localhost"
-  port = 5432
-
-  [client]
-  ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
-`,
-	Run: func(cmd *cobra.Command, args []string) {
-		SubCommand = cmd.CalledAs()
-		LogWithCommand = *log.WithField("SubCommand", SubCommand)
-		fullSync()
-	},
-}
-
-func init() {
-	rootCmd.AddCommand(fullSyncCmd)
-	fullSyncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
-}
-
-func backFillAllBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
-	populated, err := history.PopulateMissingBlocks(blockchain, blockRepository, startingBlockNumber)
-	if err != nil {
-		LogWithCommand.Error("backfillAllBlocks: error in populateMissingBlocks: ", err)
-	}
-	missingBlocksPopulated <- populated
-}
-
-func fullSync() {
-	ticker := time.NewTicker(pollingInterval)
-	defer ticker.Stop()
-
-	blockChain := getBlockChain()
-	lastBlock, err := blockChain.LastBlock()
-	if err != nil {
-		LogWithCommand.Error("fullSync: Error getting last block: ", err)
-	}
-	if lastBlock.Int64() == 0 {
-		LogWithCommand.Fatal("geth initial: state sync not finished")
-	}
-	if startingBlockNumber > lastBlock.Int64() {
-		LogWithCommand.Fatal("fullSync: starting block number > current block number")
-	}
-
-	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
-	blockRepository := repositories.NewBlockRepository(&db)
-	validator := history.NewBlockValidator(blockChain, blockRepository, validationWindow)
-	missingBlocksPopulated := make(chan int)
-	go backFillAllBlocks(blockChain, blockRepository, missingBlocksPopulated, startingBlockNumber)
-
-	for {
-		select {
-		case <-ticker.C:
-			window, err := validator.ValidateBlocks()
-			if err != nil {
-				LogWithCommand.Error("fullSync: error in validateBlocks: ", err)
-			}
-			LogWithCommand.Debug(window.GetString())
-		case <-missingBlocksPopulated:
-			go backFillAllBlocks(blockChain, blockRepository, missingBlocksPopulated, startingBlockNumber)
-		}
-	}
-}

cmd/headerSync.go

@@ -1,114 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"time"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
-	"github.com/vulcanize/vulcanizedb/pkg/history"
-	"github.com/vulcanize/vulcanizedb/utils"
-)
-
-// headerSyncCmd represents the headerSync command
-var headerSyncCmd = &cobra.Command{
-	Use:   "headerSync",
-	Short: "Syncs VulcanizeDB with local ethereum node's block headers",
-	Long: `Syncs VulcanizeDB with local ethereum node. Populates
-Postgres with block headers.
-
-./vulcanizedb headerSync --starting-block-number 0 --config public.toml
-
-Expects ethereum node to be running and requires a .toml config:
-
-  [database]
-  name = "vulcanize_public"
-  hostname = "localhost"
-  port = 5432
-
-  [client]
-  ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
-`,
-	Run: func(cmd *cobra.Command, args []string) {
-		SubCommand = cmd.CalledAs()
-		LogWithCommand = *log.WithField("SubCommand", SubCommand)
-		headerSync()
-	},
-}
-
-func init() {
-	rootCmd.AddCommand(headerSyncCmd)
-	headerSyncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
-}
-
-func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
-	populated, err := history.PopulateMissingHeaders(blockchain, headerRepository, startingBlockNumber)
-	if err != nil {
-		// TODO Lots of possible errors in the call stack above. If errors occur, we still put
-		// 0 in the channel, triggering another round
-		LogWithCommand.Error("backfillAllHeaders: Error populating headers: ", err)
-	}
-	missingBlocksPopulated <- populated
-}
-
-func headerSync() {
-	ticker := time.NewTicker(pollingInterval)
-	defer ticker.Stop()
-	blockChain := getBlockChain()
-	validateArgs(blockChain)
-	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
-
-	headerRepository := repositories.NewHeaderRepository(&db)
-	validator := history.NewHeaderValidator(blockChain, headerRepository, validationWindow)
-	missingBlocksPopulated := make(chan int)
-	go backFillAllHeaders(blockChain, headerRepository, missingBlocksPopulated, startingBlockNumber)
-
-	for {
-		select {
-		case <-ticker.C:
-			window, err := validator.ValidateHeaders()
-			if err != nil {
-				LogWithCommand.Error("headerSync: ValidateHeaders failed: ", err)
-			}
-			LogWithCommand.Debug(window.GetString())
-		case n := <-missingBlocksPopulated:
-			if n == 0 {
-				time.Sleep(3 * time.Second)
-			}
-			go backFillAllHeaders(blockChain, headerRepository, missingBlocksPopulated, startingBlockNumber)
-		}
-	}
-}
-
-func validateArgs(blockChain *geth.BlockChain) {
-	lastBlock, err := blockChain.LastBlock()
-	if err != nil {
-		LogWithCommand.Error("validateArgs: Error getting last block: ", err)
-	}
-	if lastBlock.Int64() == 0 {
-		LogWithCommand.Fatal("geth initial: state sync not finished")
-	}
-	if startingBlockNumber > lastBlock.Int64() {
-		LogWithCommand.Fatal("starting block number > current block number")
-	}
-}

cmd/resync.go

@@ -0,0 +1,122 @@
+// Copyright © 2020 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/resync"
+	v "github.com/vulcanize/ipfs-chain-watcher/version"
+)
+
+// resyncCmd represents the resync command
+var resyncCmd = &cobra.Command{
+	Use:   "resync",
+	Short: "Resync historical data",
+	Long:  `Use this command to fill in sections of missing data in the super node`,
+	Run: func(cmd *cobra.Command, args []string) {
+		subCommand = cmd.CalledAs()
+		logWithCommand = *log.WithField("SubCommand", subCommand)
+		rsyncCmdCommand()
+	},
+}
+
+func rsyncCmdCommand() {
+	logWithCommand.Infof("running vdb version: %s", v.VersionWithMeta)
+	logWithCommand.Debug("loading super node configuration variables")
+	rConfig, err := resync.NewReSyncConfig()
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+	logWithCommand.Infof("resync config: %+v", rConfig)
+	if rConfig.IPFSMode == shared.LocalInterface {
+		if err := ipfs.InitIPFSPlugins(); err != nil {
+			logWithCommand.Fatal(err)
+		}
+	}
+	logWithCommand.Debug("initializing new resync service")
+	rService, err := resync.NewResyncService(rConfig)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+	logWithCommand.Info("starting up resync process")
+	if err := rService.Resync(); err != nil {
+		logWithCommand.Fatal(err)
+	}
+	logWithCommand.Infof("%s %s resync finished", rConfig.Chain.String(), rConfig.ResyncType.String())
+}
+
+func init() {
+	rootCmd.AddCommand(resyncCmd)
+
+	// flags
+	resyncCmd.PersistentFlags().String("ipfs-path", "", "ipfs repository path")
+
+	resyncCmd.PersistentFlags().String("resync-chain", "", "which chain to support, options are currently Ethereum or Bitcoin.")
+	resyncCmd.PersistentFlags().String("resync-type", "", "which type of data to resync")
+	resyncCmd.PersistentFlags().Int("resync-start", 0, "block height to start resync")
+	resyncCmd.PersistentFlags().Int("resync-stop", 0, "block height to stop resync")
+	resyncCmd.PersistentFlags().Int("resync-batch-size", 0, "data fetching batch size")
+	resyncCmd.PersistentFlags().Int("resync-batch-number", 0, "how many goroutines to fetch data concurrently")
+	resyncCmd.PersistentFlags().Bool("resync-clear-old-cache", false, "if true, clear out old data of the provided type within the resync range before resyncing")
+	resyncCmd.PersistentFlags().Bool("resync-reset-validation", false, "if true, reset times_validated to 0")
+	resyncCmd.PersistentFlags().Int("resync-timeout", 15, "timeout used for resync http requests")
+
+	resyncCmd.PersistentFlags().String("btc-http-path", "", "http url for bitcoin node")
+	resyncCmd.PersistentFlags().String("btc-password", "", "password for btc node")
+	resyncCmd.PersistentFlags().String("btc-username", "", "username for btc node")
+	resyncCmd.PersistentFlags().String("btc-node-id", "", "btc node id")
+	resyncCmd.PersistentFlags().String("btc-client-name", "", "btc client name")
+	resyncCmd.PersistentFlags().String("btc-genesis-block", "", "btc genesis block hash")
+	resyncCmd.PersistentFlags().String("btc-network-id", "", "btc network id")
+
+	resyncCmd.PersistentFlags().String("eth-http-path", "", "http url for ethereum node")
+	resyncCmd.PersistentFlags().String("eth-node-id", "", "eth node id")
+	resyncCmd.PersistentFlags().String("eth-client-name", "", "eth client name")
+	resyncCmd.PersistentFlags().String("eth-genesis-block", "", "eth genesis block hash")
+	resyncCmd.PersistentFlags().String("eth-network-id", "", "eth network id")
+
+	// and their bindings
+	viper.BindPFlag("ipfs.path", resyncCmd.PersistentFlags().Lookup("ipfs-path"))
+
+	viper.BindPFlag("resync.chain", resyncCmd.PersistentFlags().Lookup("resync-chain"))
+	viper.BindPFlag("resync.type", resyncCmd.PersistentFlags().Lookup("resync-type"))
+	viper.BindPFlag("resync.start", resyncCmd.PersistentFlags().Lookup("resync-start"))
+	viper.BindPFlag("resync.stop", resyncCmd.PersistentFlags().Lookup("resync-stop"))
+	viper.BindPFlag("resync.batchSize", resyncCmd.PersistentFlags().Lookup("resync-batch-size"))
+	viper.BindPFlag("resync.batchNumber", resyncCmd.PersistentFlags().Lookup("resync-batch-number"))
+	viper.BindPFlag("resync.clearOldCache", resyncCmd.PersistentFlags().Lookup("resync-clear-old-cache"))
+	viper.BindPFlag("resync.resetValidation", resyncCmd.PersistentFlags().Lookup("resync-reset-validation"))
+	viper.BindPFlag("resync.timeout", resyncCmd.PersistentFlags().Lookup("resync-timeout"))
+
+	viper.BindPFlag("bitcoin.httpPath", resyncCmd.PersistentFlags().Lookup("btc-http-path"))
+	viper.BindPFlag("bitcoin.pass", resyncCmd.PersistentFlags().Lookup("btc-password"))
+	viper.BindPFlag("bitcoin.user", resyncCmd.PersistentFlags().Lookup("btc-username"))
+	viper.BindPFlag("bitcoin.nodeID", resyncCmd.PersistentFlags().Lookup("btc-node-id"))
+	viper.BindPFlag("bitcoin.clientName", resyncCmd.PersistentFlags().Lookup("btc-client-name"))
+	viper.BindPFlag("bitcoin.genesisBlock", resyncCmd.PersistentFlags().Lookup("btc-genesis-block"))
+	viper.BindPFlag("bitcoin.networkID", resyncCmd.PersistentFlags().Lookup("btc-network-id"))
+
+	viper.BindPFlag("ethereum.httpPath", resyncCmd.PersistentFlags().Lookup("eth-http-path"))
+	viper.BindPFlag("ethereum.nodeID", resyncCmd.PersistentFlags().Lookup("eth-node-id"))
+	viper.BindPFlag("ethereum.clientName", resyncCmd.PersistentFlags().Lookup("eth-client-name"))
+	viper.BindPFlag("ethereum.genesisBlock", resyncCmd.PersistentFlags().Lookup("eth-genesis-block"))
+	viper.BindPFlag("ethereum.networkID", resyncCmd.PersistentFlags().Lookup("eth-network-id"))
+}

cmd/root.go

@@ -18,36 +18,23 @@ package cmd
 
 import (
 	"fmt"
+	"os"
 	"strings"
 	"time"
 
-	"github.com/ethereum/go-ethereum/ethclient"
-	"github.com/ethereum/go-ethereum/rpc"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
-	vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/config"
 )
 
 var (
-	cfgFile              string
-	databaseConfig       config.Database
-	genConfig            config.Plugin
-	ipc                  string
-	levelDbPath          string
-	queueRecheckInterval time.Duration
-	startingBlockNumber  int64
-	storageDiffsPath     string
-	syncAll              bool
-	endingBlockNumber    int64
-	recheckHeadersArg    bool
-	SubCommand           string
-	LogWithCommand       log.Entry
+	cfgFile        string
+	databaseConfig config.Database
+	ipc            string
+	subCommand     string
+	logWithCommand log.Entry
 )
 
 const (
@@ -69,17 +56,27 @@ func Execute() {
 
 func initFuncs(cmd *cobra.Command, args []string) {
 	setViperConfigs()
-	logLvlErr := logLevel()
-	if logLvlErr != nil {
-		log.Fatal("Could not set log level: ", logLvlErr)
+	logfile := viper.GetString("logfile")
+	if logfile != "" {
+		file, err := os.OpenFile(logfile,
+			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
+		if err == nil {
+			log.Infof("Directing output to %s", logfile)
+			log.SetOutput(file)
+		} else {
+			log.SetOutput(os.Stdout)
+			log.Info("Failed to log to file, using default stdout")
+		}
+	} else {
+		log.SetOutput(os.Stdout)
+	}
+	if err := logLevel(); err != nil {
+		log.Fatal("Could not set log level: ", err)
 	}
-
 }
 
 func setViperConfigs() {
 	ipc = viper.GetString("client.ipcpath")
-	levelDbPath = viper.GetString("client.leveldbpath")
-	storageDiffsPath = viper.GetString("filesystem.storageDiffsPath")
 	databaseConfig = config.Database{
 		Name:     viper.GetString("database.name"),
 		Hostname: viper.GetString("database.hostname"),
@@ -110,57 +107,34 @@ func init() {
 	viper.AutomaticEnv()
 
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
+	rootCmd.PersistentFlags().String("logfile", "", "file path for logging")
 	rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
 	rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
 	rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
 	rootCmd.PersistentFlags().String("database-user", "", "database user")
 	rootCmd.PersistentFlags().String("database-password", "", "database password")
 	rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
-	rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
-	rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
-	rootCmd.PersistentFlags().String("exporter-name", "exporter", "name of exporter plugin")
 	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
 
+	viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile"))
 	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
 	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
 	viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
 	viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
 	viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
 	viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath"))
-	viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
-	viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
-	viper.BindPFlag("exporter.fileName", rootCmd.PersistentFlags().Lookup("exporter-name"))
 	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
 }
 
 func initConfig() {
 	if cfgFile != "" {
 		viper.SetConfigFile(cfgFile)
+		if err := viper.ReadInConfig(); err == nil {
+			log.Printf("Using config file: %s", viper.ConfigFileUsed())
+		} else {
+			log.Fatal(fmt.Sprintf("Couldn't read config file: %s", err.Error()))
+		}
 	} else {
-		noConfigError := "No config file passed with --config flag"
-		fmt.Println("Error: ", noConfigError)
-		log.Fatal(noConfigError)
-	}
-
-	if err := viper.ReadInConfig(); err == nil {
-		log.Printf("Using config file: %s\n\n", viper.ConfigFileUsed())
-	} else {
-		invalidConfigError := "Couldn't read config file"
-		formattedError := fmt.Sprintf("%s: %s", invalidConfigError, err.Error())
-		log.Fatal(formattedError)
+		log.Warn("No config file passed with --config flag")
 	}
 }
-
-func getBlockChain() *geth.BlockChain {
-	rawRpcClient, err := rpc.Dial(ipc)
-
-	if err != nil {
-		LogWithCommand.Fatal(err)
-	}
-	rpcClient := client.NewRpcClient(rawRpcClient, ipc)
-	ethClient := ethclient.NewClient(rawRpcClient)
-	vdbEthClient := client.NewEthClient(ethClient)
-	vdbNode := node.MakeNode(rpcClient)
-	transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
-	return geth.NewBlockChain(vdbEthClient, rpcClient, vdbNode, transactionConverter)
-}

cmd/streamEthSubscribe.go

@@ -0,0 +1,180 @@
+// Copyright © 2019 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/rpc"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/client"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/core"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/eth"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/streamer"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/watcher"
+)
+
+// streamEthSubscriptionCmd represents the streamEthSubscription command
+var streamEthSubscriptionCmd = &cobra.Command{
+	Use:   "streamEthSubscription",
+	Short: "This command is used to subscribe to the super node eth stream with the provided filters",
+	Long: `This command is for demo and testing purposes and is used to subscribe to the super node with the provided subscription configuration parameters.
+It does not do anything with the data streamed from the super node other than unpack it and print it out for demonstration purposes.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		subCommand = cmd.CalledAs()
+		logWithCommand = *log.WithField("SubCommand", subCommand)
+		streamEthSubscription()
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(streamEthSubscriptionCmd)
+}
+
+func streamEthSubscription() {
+	// Prep the subscription config/filters to be sent to the server
+	ethSubConfig, err := eth.NewEthSubscriptionConfig()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Create a new rpc client and a subscription streamer with that client
+	rpcClient := getRPCClient()
+	str := streamer.NewSuperNodeStreamer(rpcClient)
+
+	// Buffered channel for reading subscription payloads
+	payloadChan := make(chan watcher.SubscriptionPayload, 20000)
+
+	// Subscribe to the super node service with the given config/filter parameters
+	rlpParams, err := rlp.EncodeToBytes(ethSubConfig)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+	sub, err := str.Stream(payloadChan, rlpParams)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+	logWithCommand.Info("awaiting payloads")
+	// Receive response payloads and print out the results
+	for {
+		select {
+		case payload := <-payloadChan:
+			if payload.Err != "" {
+				logWithCommand.Error(payload.Err)
+				continue
+			}
+			var ethData eth.IPLDs
+			if err := rlp.DecodeBytes(payload.Data, &ethData); err != nil {
+				logWithCommand.Error(err)
+				continue
+			}
+			var header types.Header
+			err = rlp.Decode(bytes.NewBuffer(ethData.Header.Data), &header)
+			if err != nil {
+				logWithCommand.Error(err)
+				continue
+			}
+			fmt.Printf("Header number %d, hash %s\n", header.Number.Int64(), header.Hash().Hex())
+			fmt.Printf("header: %v\n", header)
+			for _, trxRlp := range ethData.Transactions {
+				var trx types.Transaction
+				buff := bytes.NewBuffer(trxRlp.Data)
+				stream := rlp.NewStream(buff, 0)
+				err := trx.DecodeRLP(stream)
+				if err != nil {
+					logWithCommand.Error(err)
+					continue
+				}
+				fmt.Printf("Transaction with hash %s\n", trx.Hash().Hex())
+				fmt.Printf("trx: %v\n", trx)
+			}
+			for _, rctRlp := range ethData.Receipts {
+				var rct types.Receipt
+				buff := bytes.NewBuffer(rctRlp.Data)
+				stream := rlp.NewStream(buff, 0)
+				err = rct.DecodeRLP(stream)
+				if err != nil {
+					logWithCommand.Error(err)
+					continue
+				}
+				fmt.Printf("Receipt with block hash %s, trx hash %s\n", rct.BlockHash.Hex(), rct.TxHash.Hex())
+				fmt.Printf("rct: %v\n", rct)
+				for _, l := range rct.Logs {
+					if len(l.Topics) < 1 {
+						logWithCommand.Error(fmt.Sprintf("log only has %d topics", len(l.Topics)))
+						continue
+					}
+					fmt.Printf("Log for block hash %s, trx hash %s, address %s, and with topic0 %s\n",
+						l.BlockHash.Hex(), l.TxHash.Hex(), l.Address.Hex(), l.Topics[0].Hex())
+					fmt.Printf("log: %v\n", l)
+				}
+			}
+			// This assumes leafs only
+			for _, stateNode := range ethData.StateNodes {
+				var acct state.Account
+				err = rlp.DecodeBytes(stateNode.IPLD.Data, &acct)
+				if err != nil {
+					logWithCommand.Error(err)
+					continue
+				}
+				fmt.Printf("Account for key %s, and root %s, with balance %s\n",
+					stateNode.StateLeafKey.Hex(), acct.Root.Hex(), acct.Balance.String())
+				fmt.Printf("state account: %+v\n", acct)
+			}
+			for _, storageNode := range ethData.StorageNodes {
+				fmt.Printf("Storage for state key %s ", storageNode.StateLeafKey.Hex())
+				fmt.Printf("with storage key %s\n", storageNode.StorageLeafKey.Hex())
+				var i []interface{}
+				err := rlp.DecodeBytes(storageNode.IPLD.Data, &i)
+				if err != nil {
+					logWithCommand.Error(err)
+					continue
+				}
+				// if a value node
+				if len(i) == 1 {
+					valueBytes, ok := i[0].([]byte)
+					if !ok {
+						continue
+					}
+					fmt.Printf("Storage leaf key: %s, and value hash: %s\n",
+						storageNode.StorageLeafKey.Hex(), common.BytesToHash(valueBytes).Hex())
+				}
+			}
+		case err = <-sub.Err():
+			logWithCommand.Fatal(err)
+		}
+	}
+}
+
+func getRPCClient() core.RPCClient {
+	vulcPath := viper.GetString("superNode.ethSubscription.wsPath")
+	if vulcPath == "" {
+		vulcPath = "ws://127.0.0.1:8080" // default to and try the default ws url if no path is provided
+	}
+	rawRPCClient, err := rpc.Dial(vulcPath)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+	return client.NewRPCClient(rawRPCClient, vulcPath)
+}

cmd/version.go

@@ -0,0 +1,41 @@
+// Copyright © 2020 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	v "github.com/vulcanize/ipfs-chain-watcher/version"
+)
+
+// versionCmd represents the version command
+var versionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "Prints the version of ipfs-chain-watcher",
+	Long: `Use this command to fetch the version of ipfs-chain-watcher
+
+Usage: ./ipfs-chain-watcher version`,
+	Run: func(cmd *cobra.Command, args []string) {
+		subCommand = cmd.CalledAs()
+		logWithCommand = *log.WithField("SubCommand", subCommand)
+		logWithCommand.Infof("ipfs-chain-watcher version: %s", v.VersionWithMeta)
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(versionCmd)
+}

cmd/watch.go

@@ -0,0 +1,194 @@
+// Copyright © 2020 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"os"
+	"os/signal"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/rpc"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/watcher"
+	v "github.com/vulcanize/ipfs-chain-watcher/version"
+)
+
+// superNodeCmd represents the superNode command
+var superNodeCmd = &cobra.Command{
+	Use:   "superNode",
+	Short: "sync chain data into PG-IPFS",
+	Long: `This command configures a VulcanizeDB ipfs-chain-watcher.
+
+The Sync process streams all chain data from the appropriate chain, processes this data into IPLD objects
+and publishes them to IPFS. It then indexes the CIDs against useful data fields/metadata in Postgres. 
+
+The Serve process creates and exposes a rpc subscription server over ws and ipc. Transformers can subscribe to
+these endpoints to stream
+
+The BackFill process spins up a background process which periodically probes the Postgres database to identify
+and fill in gaps in the data
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		subCommand = cmd.CalledAs()
+		logWithCommand = *log.WithField("SubCommand", subCommand)
+		superNode()
+	},
+}
+
+func superNode() {
+	logWithCommand.Infof("running vdb version: %s", v.VersionWithMeta)
+	logWithCommand.Debug("loading super node configuration variables")
+	superNodeConfig, err := watcher.NewSuperNodeConfig()
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+	logWithCommand.Infof("super node config: %+v", superNodeConfig)
+	if superNodeConfig.IPFSMode == shared.LocalInterface {
+		if err := ipfs.InitIPFSPlugins(); err != nil {
+			logWithCommand.Fatal(err)
+		}
+	}
+	wg := &sync.WaitGroup{}
+	logWithCommand.Debug("initializing new super node service")
+	superNode, err := watcher.NewSuperNode(superNodeConfig)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+	var forwardPayloadChan chan shared.ConvertedData
+	if superNodeConfig.Serve {
+		logWithCommand.Info("starting up super node servers")
+		forwardPayloadChan = make(chan shared.ConvertedData, watcher.PayloadChanBufferSize)
+		superNode.Serve(wg, forwardPayloadChan)
+		if err := startServers(superNode, superNodeConfig); err != nil {
+			logWithCommand.Fatal(err)
+		}
+	}
+	if superNodeConfig.Sync {
+		logWithCommand.Info("starting up super node sync process")
+		if err := superNode.Sync(wg, forwardPayloadChan); err != nil {
+			logWithCommand.Fatal(err)
+		}
+	}
+	var backFiller watcher.BackFillInterface
+	if superNodeConfig.BackFill {
+		logWithCommand.Debug("initializing new super node backfill service")
+		backFiller, err = watcher.NewBackFillService(superNodeConfig, forwardPayloadChan)
+		if err != nil {
+			logWithCommand.Fatal(err)
+		}
+		logWithCommand.Info("starting up super node backfill process")
+		backFiller.BackFill(wg)
+	}
+	shutdown := make(chan os.Signal)
+	signal.Notify(shutdown, os.Interrupt)
+	<-shutdown
+	if superNodeConfig.BackFill {
+		backFiller.Stop()
+	}
+	superNode.Stop()
+	wg.Wait()
+}
+
+func startServers(superNode watcher.SuperNode, settings *watcher.Config) error {
+	logWithCommand.Debug("starting up IPC server")
+	_, _, err := rpc.StartIPCEndpoint(settings.IPCEndpoint, superNode.APIs())
+	if err != nil {
+		return err
+	}
+	logWithCommand.Debug("starting up WS server")
+	_, _, err = rpc.StartWSEndpoint(settings.WSEndpoint, superNode.APIs(), []string{"vdb"}, nil, true)
+	if err != nil {
+		return err
+	}
+	logWithCommand.Debug("starting up HTTP server")
+	_, _, err = rpc.StartHTTPEndpoint(settings.HTTPEndpoint, superNode.APIs(), []string{settings.Chain.API()}, nil, nil, rpc.HTTPTimeouts{})
+	return err
+}
+
+func init() {
+	rootCmd.AddCommand(superNodeCmd)
+
+	// flags for all config variables
+	superNodeCmd.PersistentFlags().String("ipfs-path", "", "ipfs repository path")
+
+	superNodeCmd.PersistentFlags().String("supernode-chain", "", "which chain to support, options are currently Ethereum or Bitcoin.")
+	superNodeCmd.PersistentFlags().Bool("supernode-server", false, "turn vdb server on or off")
+	superNodeCmd.PersistentFlags().String("supernode-ws-path", "", "vdb server ws path")
+	superNodeCmd.PersistentFlags().String("supernode-http-path", "", "vdb server http path")
+	superNodeCmd.PersistentFlags().String("supernode-ipc-path", "", "vdb server ipc path")
+	superNodeCmd.PersistentFlags().Bool("supernode-sync", false, "turn vdb sync on or off")
+	superNodeCmd.PersistentFlags().Int("supernode-workers", 0, "how many worker goroutines to publish and index data")
+	superNodeCmd.PersistentFlags().Bool("supernode-back-fill", false, "turn vdb backfill on or off")
+	superNodeCmd.PersistentFlags().Int("supernode-frequency", 0, "how often (in seconds) the backfill process checks for gaps")
+	superNodeCmd.PersistentFlags().Int("supernode-batch-size", 0, "data fetching batch size")
+	superNodeCmd.PersistentFlags().Int("supernode-batch-number", 0, "how many goroutines to fetch data concurrently")
+	superNodeCmd.PersistentFlags().Int("supernode-validation-level", 0, "backfill will resync any data below this level")
+	superNodeCmd.PersistentFlags().Int("supernode-timeout", 0, "timeout used for backfill http requests")
+
+	superNodeCmd.PersistentFlags().String("btc-ws-path", "", "ws url for bitcoin node")
+	superNodeCmd.PersistentFlags().String("btc-http-path", "", "http url for bitcoin node")
+	superNodeCmd.PersistentFlags().String("btc-password", "", "password for btc node")
+	superNodeCmd.PersistentFlags().String("btc-username", "", "username for btc node")
+	superNodeCmd.PersistentFlags().String("btc-node-id", "", "btc node id")
+	superNodeCmd.PersistentFlags().String("btc-client-name", "", "btc client name")
+	superNodeCmd.PersistentFlags().String("btc-genesis-block", "", "btc genesis block hash")
+	superNodeCmd.PersistentFlags().String("btc-network-id", "", "btc network id")
+
+	superNodeCmd.PersistentFlags().String("eth-ws-path", "", "ws url for ethereum node")
+	superNodeCmd.PersistentFlags().String("eth-http-path", "", "http url for ethereum node")
+	superNodeCmd.PersistentFlags().String("eth-node-id", "", "eth node id")
+	superNodeCmd.PersistentFlags().String("eth-client-name", "", "eth client name")
+	superNodeCmd.PersistentFlags().String("eth-genesis-block", "", "eth genesis block hash")
+	superNodeCmd.PersistentFlags().String("eth-network-id", "", "eth network id")
+
+	// and their bindings
+	viper.BindPFlag("ipfs.path", superNodeCmd.PersistentFlags().Lookup("ipfs-path"))
+
+	viper.BindPFlag("superNode.chain", superNodeCmd.PersistentFlags().Lookup("supernode-chain"))
+	viper.BindPFlag("superNode.server", superNodeCmd.PersistentFlags().Lookup("supernode-server"))
+	viper.BindPFlag("superNode.wsPath", superNodeCmd.PersistentFlags().Lookup("supernode-ws-path"))
+	viper.BindPFlag("superNode.httpPath", superNodeCmd.PersistentFlags().Lookup("supernode-http-path"))
+	viper.BindPFlag("superNode.ipcPath", superNodeCmd.PersistentFlags().Lookup("supernode-ipc-path"))
+	viper.BindPFlag("superNode.sync", superNodeCmd.PersistentFlags().Lookup("supernode-sync"))
+	viper.BindPFlag("superNode.workers", superNodeCmd.PersistentFlags().Lookup("supernode-workers"))
+	viper.BindPFlag("superNode.backFill", superNodeCmd.PersistentFlags().Lookup("supernode-back-fill"))
+	viper.BindPFlag("superNode.frequency", superNodeCmd.PersistentFlags().Lookup("supernode-frequency"))
+	viper.BindPFlag("superNode.batchSize", superNodeCmd.PersistentFlags().Lookup("supernode-batch-size"))
+	viper.BindPFlag("superNode.batchNumber", superNodeCmd.PersistentFlags().Lookup("supernode-batch-number"))
+	viper.BindPFlag("superNode.validationLevel", superNodeCmd.PersistentFlags().Lookup("supernode-validation-level"))
+	viper.BindPFlag("superNode.timeout", superNodeCmd.PersistentFlags().Lookup("supernode-timeout"))
+
+	viper.BindPFlag("bitcoin.wsPath", superNodeCmd.PersistentFlags().Lookup("btc-ws-path"))
+	viper.BindPFlag("bitcoin.httpPath", superNodeCmd.PersistentFlags().Lookup("btc-http-path"))
+	viper.BindPFlag("bitcoin.pass", superNodeCmd.PersistentFlags().Lookup("btc-password"))
+	viper.BindPFlag("bitcoin.user", superNodeCmd.PersistentFlags().Lookup("btc-username"))
+	viper.BindPFlag("bitcoin.nodeID", superNodeCmd.PersistentFlags().Lookup("btc-node-id"))
+	viper.BindPFlag("bitcoin.clientName", superNodeCmd.PersistentFlags().Lookup("btc-client-name"))
+	viper.BindPFlag("bitcoin.genesisBlock", superNodeCmd.PersistentFlags().Lookup("btc-genesis-block"))
+	viper.BindPFlag("bitcoin.networkID", superNodeCmd.PersistentFlags().Lookup("btc-network-id"))
+
+	viper.BindPFlag("ethereum.wsPath", superNodeCmd.PersistentFlags().Lookup("eth-ws-path"))
+	viper.BindPFlag("ethereum.httpPath", superNodeCmd.PersistentFlags().Lookup("eth-http-path"))
+	viper.BindPFlag("ethereum.nodeID", superNodeCmd.PersistentFlags().Lookup("eth-node-id"))
+	viper.BindPFlag("ethereum.clientName", superNodeCmd.PersistentFlags().Lookup("eth-client-name"))
+	viper.BindPFlag("ethereum.genesisBlock", superNodeCmd.PersistentFlags().Lookup("eth-genesis-block"))
+	viper.BindPFlag("ethereum.networkID", superNodeCmd.PersistentFlags().Lookup("eth-network-id"))
+}

db/migrations/00001_create_blocks_table.sql

@@ -1,23 +0,0 @@
--- +goose Up
-CREATE TABLE public.blocks (
-  id            SERIAL PRIMARY KEY,
-  difficulty    BIGINT,
-  extra_data    VARCHAR,
-  gas_limit      BIGINT,
-  gas_used       BIGINT,
-  hash          VARCHAR(66),
-  miner         VARCHAR(42),
-  nonce         VARCHAR(20),
-  "number"      BIGINT,
-  parent_hash    VARCHAR(66),
-  reward        NUMERIC,
-  uncles_reward NUMERIC,
-  "size"        VARCHAR,
-  "time"        BIGINT,
-  is_final      BOOLEAN,
-  uncle_hash    VARCHAR(66)
-);
-
-
--- +goose Down
-DROP TABLE public.blocks;

db/migrations/00001_create_nodes_table.sql

@@ -3,7 +3,7 @@ CREATE TABLE nodes (
   id            SERIAL PRIMARY KEY,
   client_name   VARCHAR,
   genesis_block VARCHAR(66),
-  network_id    NUMERIC,
+  network_id    VARCHAR,
   node_id       VARCHAR(128),
   CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id)
 );

db/migrations/00002_create_eth_schema.sql

@@ -0,0 +1,5 @@
+-- +goose Up
+CREATE SCHEMA eth;
+
+-- +goose Down
+DROP SCHEMA eth;

db/migrations/00002_create_full_sync_transactions_table.sql

@@ -1,18 +0,0 @@
--- +goose Up
-CREATE TABLE full_sync_transactions (
-  id          SERIAL PRIMARY KEY,
-  block_id    INTEGER NOT NULL REFERENCES blocks(id) ON DELETE CASCADE,
-  gas_limit    NUMERIC,
-  gas_price    NUMERIC,
-  hash        VARCHAR(66),
-  input_data  BYTEA,
-  nonce       NUMERIC,
-  raw         BYTEA,
-  tx_from     VARCHAR(66),
-  tx_index    INTEGER,
-  tx_to       VARCHAR(66),
-  "value"     NUMERIC
-);
-
--- +goose Down
-DROP TABLE full_sync_transactions;

db/migrations/00003_add_block_index_to_blocks.sql

@@ -1,6 +0,0 @@
--- +goose Up
-CREATE INDEX number_index ON blocks (number);
-
-
--- +goose Down
-DROP INDEX number_index;

db/migrations/00003_create_eth_header_cids_table.sql

@@ -0,0 +1,15 @@
+-- +goose Up
+CREATE TABLE eth.header_cids (
+  id                    SERIAL PRIMARY KEY,
+  block_number          BIGINT NOT NULL,
+  block_hash            VARCHAR(66) NOT NULL,
+  parent_hash           VARCHAR(66) NOT NULL,
+  cid                   TEXT NOT NULL,
+  td                    NUMERIC NOT NULL,
+  node_id               INTEGER NOT NULL REFERENCES nodes (id) ON DELETE CASCADE,
+  reward                NUMERIC NOT NULL,
+  UNIQUE (block_number, block_hash)
+);
+
+-- +goose Down
+DROP TABLE eth.header_cids;

db/migrations/00004_create_contracts_table.sql

@@ -1,10 +0,0 @@
--- +goose Up
-CREATE TABLE watched_contracts
-(
-  contract_id   SERIAL PRIMARY KEY,
-  contract_abi  json,
-  contract_hash VARCHAR(66) UNIQUE
-);
-
--- +goose Down
-DROP TABLE watched_contracts;

db/migrations/00004_create_eth_uncle_cids_table.sql

@@ -0,0 +1,13 @@
+-- +goose Up
+CREATE TABLE eth.uncle_cids (
+  id                    SERIAL PRIMARY KEY,
+  header_id             INTEGER NOT NULL REFERENCES eth.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+  block_hash            VARCHAR(66) NOT NULL,
+  parent_hash           VARCHAR(66) NOT NULL,
+  cid                   TEXT NOT NULL,
+  reward                NUMERIC NOT NULL,
+  UNIQUE (header_id, block_hash)
+);
+
+-- +goose Down
+DROP TABLE eth.uncle_cids;

db/migrations/00005_create_eth_transaction_cids_table.sql

@@ -0,0 +1,14 @@
+-- +goose Up
+CREATE TABLE eth.transaction_cids (
+  id                    SERIAL PRIMARY KEY,
+  header_id             INTEGER NOT NULL REFERENCES eth.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+  tx_hash               VARCHAR(66) NOT NULL,
+  index                 INTEGER NOT NULL,
+  cid                   TEXT NOT NULL,
+  dst                   VARCHAR(66) NOT NULL,
+  src                   VARCHAR(66) NOT NULL,
+  UNIQUE (header_id, tx_hash)
+);
+
+-- +goose Down
+DROP TABLE eth.transaction_cids;

db/migrations/00006_add_node_fk_to_blocks.sql

@@ -1,11 +0,0 @@
--- +goose Up
-ALTER TABLE blocks
-  ADD COLUMN node_id INTEGER NOT NULL,
-  ADD CONSTRAINT node_fk
-FOREIGN KEY (node_id)
-REFERENCES nodes (id)
-ON DELETE CASCADE;
-
--- +goose Down
-ALTER TABLE blocks
-  DROP COLUMN node_id;

db/migrations/00006_create_eth_receipt_cids_table.sql

@@ -0,0 +1,14 @@
+-- +goose Up
+CREATE TABLE eth.receipt_cids (
+  id                    SERIAL PRIMARY KEY,
+  tx_id                 INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+  cid                   TEXT NOT NULL,
+  contract              VARCHAR(66),
+  topic0s               VARCHAR(66)[],
+  topic1s               VARCHAR(66)[],
+  topic2s               VARCHAR(66)[],
+  topic3s               VARCHAR(66)[]
+);
+
+-- +goose Down
+DROP TABLE eth.receipt_cids;

db/migrations/00007_create_eth_state_cids_table.sql

@@ -0,0 +1,12 @@
+-- +goose Up
+CREATE TABLE eth.state_cids (
+  id                    SERIAL PRIMARY KEY,
+  header_id             INTEGER NOT NULL REFERENCES eth.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+  state_key             VARCHAR(66) NOT NULL,
+  leaf                  BOOLEAN NOT NULL,
+  cid                   TEXT NOT NULL,
+  UNIQUE (header_id, state_key)
+);
+
+-- +goose Down
+DROP TABLE eth.state_cids;

db/migrations/00007_create_full_sync_logs_table.sql

@@ -1,18 +0,0 @@
--- +goose Up
-CREATE TABLE full_sync_logs
-(
-    id           SERIAL PRIMARY KEY,
-    block_number BIGINT,
-    address      VARCHAR(66),
-    tx_hash      VARCHAR(66),
-    index        BIGINT,
-    topic0       VARCHAR(66),
-    topic1       VARCHAR(66),
-    topic2       VARCHAR(66),
-    topic3       VARCHAR(66),
-    data         TEXT
-);
-
-
--- +goose Down
-DROP TABLE full_sync_logs;

db/migrations/00008_add_node_block_number_unique_constraint_to_blocks.sql

@@ -1,7 +0,0 @@
--- +goose Up
-ALTER TABLE blocks
-  ADD CONSTRAINT node_id_block_number_uc UNIQUE (number, node_id);
-
--- +goose Down
-ALTER TABLE blocks
-  DROP CONSTRAINT node_id_block_number_uc;

db/migrations/00008_create_eth_storage_cids_table.sql

@@ -0,0 +1,12 @@
+-- +goose Up
+CREATE TABLE eth.storage_cids (
+  id                    SERIAL PRIMARY KEY,
+  state_id              INTEGER NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+  storage_key           VARCHAR(66) NOT NULL,
+  leaf                  BOOLEAN NOT NULL,
+  cid                   TEXT NOT NULL,
+  UNIQUE (state_id, storage_key)
+);
+
+-- +goose Down
+DROP TABLE eth.storage_cids;

db/migrations/00009_add_block_id_index_to_full_sync_transactions.sql

@@ -1,5 +0,0 @@
--- +goose Up
-CREATE INDEX block_id_index ON full_sync_transactions (block_id);
-
--- +goose Down
-DROP INDEX block_id_index;

db/migrations/00009_create_ipfs_blocks_table.sql

@@ -0,0 +1,8 @@
+-- +goose Up
+CREATE TABLE IF NOT EXISTS public.blocks (
+  key TEXT UNIQUE NOT NULL,
+  data BYTEA NOT NULL
+);
+
+-- +goose Down
+DROP TABLE public.blocks;

db/migrations/00010_add_node_id_index_to_blocks.sql

@@ -1,5 +0,0 @@
--- +goose Up
-CREATE INDEX node_id_index ON blocks (node_id);
-
--- +goose Down
-DROP INDEX node_id_index;

db/migrations/00010_create_btc_schema.sql

@@ -0,0 +1,5 @@
+-- +goose Up
+CREATE SCHEMA btc;
+
+-- +goose Down
+DROP SCHEMA btc;

db/migrations/00011_add_tx_to_index_to_full_sync_transactions.sql

@@ -1,5 +0,0 @@
--- +goose Up
-CREATE INDEX tx_to_index ON full_sync_transactions(tx_to);
-
--- +goose Down
-DROP INDEX tx_to_index;

db/migrations/00011_create_btc_header_cids_table.sql

@@ -0,0 +1,15 @@
+-- +goose Up
+CREATE TABLE btc.header_cids (
+	id           SERIAL  PRIMARY KEY,
+	block_number BIGINT NOT NULL,
+	block_hash   VARCHAR(66) NOT NULL,
+	parent_hash  VARCHAR(66) NOT NULL,
+	cid          TEXT NOT NULL,
+	timestamp    NUMERIC NOT NULL,
+	bits         BIGINT NOT NULL,
+	node_id      INTEGER NOT NULL REFERENCES nodes (id) ON DELETE CASCADE,
+	UNIQUE (block_number, block_hash)
+);
+
+-- +goose Down
+DROP TABLE btc.header_cids;

db/migrations/00012_add_tx_from_index_to_full_sync_transactions.sql

@@ -1,5 +0,0 @@
--- +goose Up
-CREATE INDEX tx_from_index ON full_sync_transactions(tx_from);
-
--- +goose Down
-DROP INDEX tx_from_index;

db/migrations/00012_create_btc_transaction_cids_table.sql

@@ -0,0 +1,13 @@
+-- +goose Up
+CREATE TABLE btc.transaction_cids (
+  id           SERIAL PRIMARY KEY,
+	header_id    INTEGER NOT NULL REFERENCES btc.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+	index        INTEGER NOT NULL,
+	tx_hash      VARCHAR(66) NOT NULL UNIQUE,
+	cid          TEXT NOT NULL,
+	segwit       BOOL NOT NULL,
+	witness_hash VARCHAR(66)
+);
+
+-- +goose Down
+DROP TABLE btc.transaction_cids;

db/migrations/00013_add_address_table.sql

@@ -1,10 +0,0 @@
--- +goose Up
-CREATE TABLE public.addresses
-(
-    id      SERIAL PRIMARY KEY,
-    address character varying(42),
-    UNIQUE (address)
-);
-
--- +goose Down
-DROP TABLE public.addresses;

db/migrations/00013_create_btc_tx_outputs_table.sql

@@ -0,0 +1,15 @@
+-- +goose Up
+CREATE TABLE btc.tx_outputs (
+  id            SERIAL PRIMARY KEY,
+	tx_id         INTEGER NOT NULL REFERENCES btc.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+	index         INTEGER NOT NULL,
+	value         BIGINT NOT NULL,
+	pk_script     BYTEA NOT NULL,
+	script_class  INTEGER NOT NULL,
+	addresses     VARCHAR(66)[],
+	required_sigs INTEGER NOT NULL,
+	UNIQUE (tx_id, index)
+);
+
+-- +goose Down
+DROP TABLE btc.tx_outputs;

db/migrations/00014_create_btc_tx_inputs_table.sql

@@ -0,0 +1,14 @@
+-- +goose Up
+CREATE TABLE btc.tx_inputs (
+  id               SERIAL PRIMARY KEY,
+	tx_id            INTEGER NOT NULL REFERENCES btc.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+	index            INTEGER NOT NULL,
+	witness          VARCHAR[],
+	sig_script       BYTEA NOT NULL,
+	outpoint_tx_hash VARCHAR(66) NOT NULL,
+	outpoint_index   NUMERIC NOT NULL,
+	UNIQUE (tx_id, index)
+);
+
+-- +goose Down
+DROP TABLE btc.tx_inputs;

db/migrations/00014_create_receipts_table.sql

@@ -1,16 +0,0 @@
--- +goose Up
-CREATE TABLE full_sync_receipts
-(
-    id                  SERIAL PRIMARY KEY,
-    transaction_id      INTEGER NOT NULL REFERENCES full_sync_transactions (id) ON DELETE CASCADE,
-    contract_address_id INTEGER NOT NULL REFERENCES addresses (id) ON DELETE CASCADE,
-    cumulative_gas_used NUMERIC,
-    gas_used            NUMERIC,
-    state_root          VARCHAR(66),
-    status              INTEGER,
-    tx_hash             VARCHAR(66)
-);
-
-
--- +goose Down
-DROP TABLE full_sync_receipts;

db/migrations/00015_add_transaction_id_index_to_receipts.sql

@@ -1,5 +0,0 @@
--- +goose Up
-CREATE INDEX transaction_id_index ON full_sync_receipts (transaction_id);
-
--- +goose Down
-DROP INDEX transaction_id_index;

db/migrations/00015_create_eth_queued_data_table.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+CREATE TABLE eth.queue_data (
+  id SERIAL PRIMARY KEY,
+  data BYTEA NOT NULL,
+  height BIGINT UNIQUE NOT NULL
+);
+
+-- +goose Down
+DROP TABLE eth.queue_data;

db/migrations/00016_add_receipts_fk_to_logs.sql

@@ -1,17 +0,0 @@
--- +goose Up
-ALTER TABLE full_sync_logs
-    ADD COLUMN receipt_id INT;
-
-ALTER TABLE full_sync_logs
-    ADD CONSTRAINT receipts_fk
-        FOREIGN KEY (receipt_id)
-            REFERENCES full_sync_receipts (id)
-            ON DELETE CASCADE;
-
-
--- +goose Down
-ALTER TABLE full_sync_logs
-    DROP CONSTRAINT receipts_fk;
-
-ALTER TABLE full_sync_logs
-    DROP COLUMN receipt_id;

db/migrations/00016_create_btc_queued_data_table.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+CREATE TABLE btc.queue_data (
+  id SERIAL PRIMARY KEY,
+  data BYTEA NOT NULL,
+  height BIGINT UNIQUE NOT NULL
+);
+
+-- +goose Down
+DROP TABLE btc.queue_data;

db/migrations/00017_create_log_filters.sql

@@ -1,16 +0,0 @@
--- +goose Up
-CREATE TABLE log_filters (
-  id         SERIAL,
-  name       VARCHAR NOT NULL CHECK (name <> ''),
-  from_block BIGINT CHECK (from_block >= 0),
-  to_block   BIGINT CHECK (from_block >= 0),
-  address    VARCHAR(66),
-  topic0     VARCHAR(66),
-  topic1     VARCHAR(66),
-  topic2     VARCHAR(66),
-  topic3     VARCHAR(66),
-  CONSTRAINT name_uc UNIQUE (name)
-);
-
--- +goose Down
-DROP TABLE log_filters;

db/migrations/00017_create_postgraphile_comments.sql

@@ -0,0 +1,11 @@
+-- +goose Up
+COMMENT ON TABLE public.nodes IS E'@name NodeInfo';
+COMMENT ON TABLE btc.header_cids IS E'@name BtcHeaderCids';
+COMMENT ON TABLE btc.transaction_cids IS E'@name BtcTransactionCids';
+COMMENT ON TABLE btc.queue_data IS E'@name BtcQueueData';
+COMMENT ON TABLE eth.transaction_cids IS E'@name EthTransactionCids';
+COMMENT ON TABLE eth.header_cids IS E'@name EthHeaderCids';
+COMMENT ON TABLE eth.queue_data IS E'@name EthQueueData';
+COMMENT ON COLUMN public.nodes.node_id IS E'@name ChainNodeID';
+COMMENT ON COLUMN eth.header_cids.node_id IS E'@name EthNodeID';
+COMMENT ON COLUMN btc.header_cids.node_id IS E'@name BtcNodeID';

db/migrations/00018_create_watched_event_logs.sql

@@ -1,32 +0,0 @@
--- +goose Up
-CREATE VIEW block_stats AS
-SELECT max(block_number) AS max_block,
-       min(block_number) AS min_block
-FROM full_sync_logs;
-
-CREATE VIEW watched_event_logs AS
-SELECT log_filters.name,
-       full_sync_logs.id,
-       block_number,
-       full_sync_logs.address,
-       tx_hash,
-       index,
-       full_sync_logs.topic0,
-       full_sync_logs.topic1,
-       full_sync_logs.topic2,
-       full_sync_logs.topic3,
-       data,
-       receipt_id
-FROM log_filters
-         CROSS JOIN block_stats
-         JOIN full_sync_logs ON full_sync_logs.address = log_filters.address
-    AND full_sync_logs.block_number >= coalesce(log_filters.from_block, block_stats.min_block)
-    AND full_sync_logs.block_number <= coalesce(log_filters.to_block, block_stats.max_block)
-WHERE (log_filters.topic0 = full_sync_logs.topic0 OR log_filters.topic0 ISNULL)
-  AND (log_filters.topic1 = full_sync_logs.topic1 OR log_filters.topic1 ISNULL)
-  AND (log_filters.topic2 = full_sync_logs.topic2 OR log_filters.topic2 ISNULL)
-  AND (log_filters.topic3 = full_sync_logs.topic3 OR log_filters.topic3 ISNULL);
-
--- +goose Down
-DROP VIEW watched_event_logs;
-DROP VIEW block_stats;

db/migrations/00018_update_state_cids.sql

@@ -0,0 +1,37 @@
+-- +goose Up
+ALTER TABLE eth.state_cids
+ADD COLUMN state_path BYTEA;
+
+ALTER TABLE eth.state_cids
+DROP COLUMN leaf;
+
+ALTER TABLE eth.state_cids
+ADD COLUMN node_type INTEGER;
+
+ALTER TABLE eth.state_cids
+ALTER COLUMN state_key DROP NOT NULL;
+
+ALTER TABLE eth.state_cids
+DROP CONSTRAINT state_cids_header_id_state_key_key;
+
+ALTER TABLE eth.state_cids
+ADD CONSTRAINT state_cids_header_id_state_path_key UNIQUE (header_id, state_path);
+
+-- +goose Down
+ALTER TABLE eth.state_cids
+ADD CONSTRAINT state_cids_header_id_state_key_key UNIQUE (header_id, state_key);
+
+ALTER TABLE eth.state_cids
+DROP CONSTRAINT state_cids_header_id_state_path_key;
+
+ALTER TABLE eth.state_cids
+ALTER COLUMN state_key SET NOT NULL;
+
+ALTER TABLE eth.state_cids
+DROP COLUMN node_type;
+
+ALTER TABLE eth.state_cids
+ADD COLUMN leaf BOOLEAN NOT NULL;
+
+ALTER TABLE eth.state_cids
+DROP COLUMN state_path;

db/migrations/00019_update_log_filters_to_block_constraint.sql

@@ -1,14 +0,0 @@
--- +goose Up
-ALTER TABLE log_filters
-  DROP CONSTRAINT log_filters_from_block_check1;
-
-ALTER TABLE log_filters
-  ADD CONSTRAINT log_filters_to_block_check CHECK (to_block >= 0);
-
-
--- +goose Down
-ALTER TABLE log_filters
-  DROP CONSTRAINT log_filters_to_block_check;
-
-ALTER TABLE log_filters
-  ADD CONSTRAINT log_filters_from_block_check1 CHECK (to_block >= 0);

db/migrations/00019_update_storage_cids.sql

@@ -0,0 +1,37 @@
+-- +goose Up
+ALTER TABLE eth.storage_cids
+ADD COLUMN storage_path BYTEA;
+
+ALTER TABLE eth.storage_cids
+DROP COLUMN leaf;
+
+ALTER TABLE eth.storage_cids
+ADD COLUMN node_type INTEGER;
+
+ALTER TABLE eth.storage_cids
+ALTER COLUMN storage_key DROP NOT NULL;
+
+ALTER TABLE eth.storage_cids
+DROP CONSTRAINT storage_cids_state_id_storage_key_key;
+
+ALTER TABLE eth.storage_cids
+ADD CONSTRAINT storage_cids_state_id_storage_path_key UNIQUE (state_id, storage_path);
+
+-- +goose Down
+ALTER TABLE eth.storage_cids
+DROP CONSTRAINT storage_cids_state_id_storage_path_key;
+
+ALTER TABLE eth.storage_cids
+ADD CONSTRAINT storage_cids_state_id_storage_key_key UNIQUE (state_id, storage_key);
+
+ALTER TABLE eth.storage_cids
+ALTER COLUMN storage_key SET NOT NULL;
+
+ALTER TABLE eth.storage_cids
+DROP COLUMN node_type;
+
+ALTER TABLE eth.storage_cids
+ADD COLUMN leaf BOOLEAN NOT NULL;
+
+ALTER TABLE eth.storage_cids
+DROP COLUMN storage_path;

db/migrations/00020_rename_node_table.sql

@@ -1,43 +0,0 @@
--- +goose Up
-ALTER TABLE public.nodes RENAME TO eth_nodes;
-
-ALTER TABLE public.eth_nodes RENAME COLUMN node_id TO eth_node_id;
-
-ALTER TABLE public.eth_nodes DROP CONSTRAINT node_uc;
-ALTER TABLE public.eth_nodes
-  ADD CONSTRAINT eth_node_uc UNIQUE (genesis_block, network_id, eth_node_id);
-
-ALTER TABLE public.blocks RENAME COLUMN node_id TO eth_node_id;
-
-ALTER TABLE public.blocks DROP CONSTRAINT node_id_block_number_uc;
-ALTER TABLE public.blocks
-  ADD CONSTRAINT eth_node_id_block_number_uc UNIQUE (number, eth_node_id);
-
-ALTER TABLE public.blocks DROP CONSTRAINT node_fk;
-ALTER TABLE public.blocks
-  ADD CONSTRAINT node_fk
-FOREIGN KEY (eth_node_id) REFERENCES eth_nodes (id) ON DELETE CASCADE;
-
-
--- +goose Down
-ALTER TABLE public.eth_nodes
-  RENAME TO nodes;
-
-ALTER TABLE public.nodes
-  RENAME COLUMN eth_node_id TO node_id;
-
-ALTER TABLE public.nodes
-  DROP CONSTRAINT eth_node_uc;
-ALTER TABLE public.nodes
-  ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id);
-
-ALTER TABLE public.blocks RENAME COLUMN eth_node_id TO node_id;
-
-ALTER TABLE public.blocks DROP CONSTRAINT eth_node_id_block_number_uc;
-ALTER TABLE public.blocks
-  ADD CONSTRAINT node_id_block_number_uc UNIQUE (number, node_id);
-
-ALTER TABLE public.blocks DROP CONSTRAINT node_fk;
-ALTER TABLE public.blocks
-  ADD CONSTRAINT node_fk
-FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE;

db/migrations/00020_update_header_cids.sql

@@ -0,0 +1,37 @@
+-- +goose Up
+ALTER TABLE eth.header_cids
+ADD COLUMN state_root VARCHAR(66);
+
+ALTER TABLE eth.header_cids
+ADD COLUMN tx_root VARCHAR(66);
+
+ALTER TABLE eth.header_cids
+ADD COLUMN receipt_root VARCHAR(66);
+
+ALTER TABLE eth.header_cids
+ADD COLUMN uncle_root VARCHAR(66);
+
+ALTER TABLE eth.header_cids
+ADD COLUMN bloom BYTEA;
+
+ALTER TABLE eth.header_cids
+ADD COLUMN timestamp NUMERIC;
+
+-- +goose Down
+ALTER TABLE eth.header_cids
+DROP COLUMN timestamp;
+
+ALTER TABLE eth.header_cids
+DROP COLUMN bloom;
+
+ALTER TABLE eth.header_cids
+DROP COLUMN uncle_root;
+
+ALTER TABLE eth.header_cids
+DROP COLUMN receipt_root;
+
+ALTER TABLE eth.header_cids
+DROP COLUMN tx_root;
+
+ALTER TABLE eth.header_cids
+DROP COLUMN state_root;

db/migrations/00021_associate_receipts_with_blocks.sql

@@ -1,44 +0,0 @@
--- +goose Up
-ALTER TABLE full_sync_receipts
-  ADD COLUMN block_id INT;
-
-UPDATE full_sync_receipts
-  SET block_id = (
-    SELECT block_id FROM full_sync_transactions WHERE full_sync_transactions.id = full_sync_receipts.transaction_id
-  );
-
-ALTER TABLE full_sync_receipts
-  ALTER COLUMN block_id SET NOT NULL;
-
-ALTER TABLE full_sync_receipts
-  ADD CONSTRAINT blocks_fk
-FOREIGN KEY (block_id)
-REFERENCES blocks (id)
-ON DELETE CASCADE;
-
-ALTER TABLE full_sync_receipts
-  DROP COLUMN transaction_id;
-
-
--- +goose Down
-ALTER TABLE full_sync_receipts
-  ADD COLUMN transaction_id INT;
-
-CREATE INDEX transaction_id_index ON full_sync_receipts (transaction_id);
-
-UPDATE full_sync_receipts
-  SET transaction_id = (
-    SELECT id FROM full_sync_transactions WHERE full_sync_transactions.hash = full_sync_receipts.tx_hash
-  );
-
-ALTER TABLE full_sync_receipts
-  ALTER COLUMN transaction_id SET NOT NULL;
-
-ALTER TABLE full_sync_receipts
-  ADD CONSTRAINT transaction_fk
-FOREIGN KEY (transaction_id)
-REFERENCES full_sync_transactions (id)
-ON DELETE CASCADE;
-
-ALTER TABLE full_sync_receipts
-  DROP COLUMN block_id;

db/migrations/00021_create_eth_state_accouts_table.sql

@@ -0,0 +1,13 @@
+-- +goose Up
+CREATE TABLE eth.state_accounts (
+  id                    SERIAL PRIMARY KEY,
+  state_id              INTEGER NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE,
+  balance               NUMERIC NOT NULL,
+  nonce                 INTEGER NOT NULL,
+  code_hash             BYTEA NOT NULL,
+  storage_root          VARCHAR(66) NOT NULL,
+  UNIQUE (state_id)
+);
+
+-- +goose Down
+DROP TABLE eth.state_accounts;

db/migrations/00022_add_eth_node_fingerprint_to_blocks.sql

@@ -1,16 +0,0 @@
--- +goose Up
-ALTER TABLE blocks
-  ADD COLUMN eth_node_fingerprint VARCHAR(128);
-
-UPDATE blocks
-  SET eth_node_fingerprint = (
-    SELECT eth_node_id FROM eth_nodes WHERE eth_nodes.id = blocks.eth_node_id
-  );
-
-ALTER TABLE blocks
-  ALTER COLUMN eth_node_fingerprint SET NOT NULL;
-
-
--- +goose Down
-ALTER TABLE blocks
-  DROP COLUMN eth_node_fingerprint;

db/migrations/00022_rename_to_leaf_key.sql

@@ -0,0 +1,13 @@
+-- +goose Up
+ALTER TABLE eth.state_cids
+RENAME COLUMN state_key TO state_leaf_key;
+
+ALTER TABLE eth.storage_cids
+RENAME COLUMN storage_key TO storage_leaf_key;
+
+-- +goose Down
+ALTER TABLE eth.storage_cids
+RENAME COLUMN storage_leaf_key TO storage_key;
+
+ALTER TABLE eth.state_cids
+RENAME COLUMN state_leaf_key TO state_key;

db/migrations/00023_create_headers_table.sql

@@ -1,19 +0,0 @@
--- +goose Up
-CREATE TABLE public.headers
-(
-    id                   SERIAL PRIMARY KEY,
-    hash                 VARCHAR(66),
-    block_number         BIGINT,
-    raw                  JSONB,
-    block_timestamp      NUMERIC,
-    check_count          INTEGER NOT NULL DEFAULT 0,
-    eth_node_id          INTEGER NOT NULL REFERENCES eth_nodes (id) ON DELETE CASCADE,
-    eth_node_fingerprint VARCHAR(128)
-);
-
--- Index is removed when table is
-CREATE INDEX headers_block_number ON public.headers (block_number);
-
-
--- +goose Down
-DROP TABLE public.headers;

db/migrations/00023_update_receipt_cids.sql

@@ -0,0 +1,22 @@
+-- +goose Up
+ALTER TABLE eth.receipt_cids
+ADD COLUMN log_contracts VARCHAR(66)[];
+
+ALTER TABLE eth.receipt_cids
+ADD COLUMN contract_hash VARCHAR(66);
+
+WITH uniques AS (SELECT DISTINCT ON (tx_id) * FROM eth.receipt_cids)
+DELETE FROM eth.receipt_cids WHERE receipt_cids.id NOT IN (SELECT id FROM uniques);
+
+ALTER TABLE eth.receipt_cids
+ADD CONSTRAINT receipt_cids_tx_id_key UNIQUE (tx_id);
+
+-- +goose Down
+ALTER TABLE eth.receipt_cids
+DROP CONSTRAINT receipt_cids_tx_id_key;
+
+ALTER TABLE eth.receipt_cids
+DROP COLUMN contract_hash;
+
+ALTER TABLE eth.receipt_cids
+DROP COLUMN log_contracts;

db/migrations/00024_add_times_validated.sql

@@ -0,0 +1,13 @@
+-- +goose Up
+ALTER TABLE eth.header_cids
+ADD COLUMN times_validated INTEGER NOT NULL DEFAULT 1;
+
+ALTER TABLE btc.header_cids
+ADD COLUMN times_validated INTEGER NOT NULL DEFAULT 1;
+
+-- +goose Down
+ALTER TABLE btc.header_cids
+DROP COLUMN times_validated;
+
+ALTER TABLE eth.header_cids
+DROP COLUMN times_validated;

db/migrations/00024_create_checked_headers_table.sql

@@ -1,8 +0,0 @@
--- +goose Up
-CREATE TABLE public.checked_headers (
-  id                  SERIAL PRIMARY KEY,
-  header_id           INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE
-);
-
--- +goose Down
-DROP TABLE public.checked_headers;

db/migrations/00025_create_queued_storage.sql

@@ -1,13 +0,0 @@
--- +goose Up
-CREATE TABLE public.queued_storage (
-  id            SERIAL PRIMARY KEY,
-  block_height  BIGINT,
-  block_hash    BYTEA,
-  contract      BYTEA,
-  storage_key   BYTEA,
-  storage_value BYTEA,
-  UNIQUE (block_height, block_hash, contract, storage_key, storage_value)
-);
-
--- +goose Down
-DROP TABLE public.queued_storage;

db/migrations/00026_create_header_sync_transactions_table.sql

@@ -1,19 +0,0 @@
--- +goose Up
-CREATE TABLE header_sync_transactions (
-  id          SERIAL PRIMARY KEY,
-  header_id   INTEGER NOT NULL REFERENCES headers(id) ON DELETE CASCADE,
-  hash        VARCHAR(66),
-  gas_limit   NUMERIC,
-  gas_price   NUMERIC,
-  input_data  BYTEA,
-  nonce       NUMERIC,
-  raw         BYTEA,
-  tx_from     VARCHAR(44),
-  tx_index    INTEGER,
-  tx_to       VARCHAR(44),
-  "value"     NUMERIC,
-  UNIQUE (header_id, hash)
-);
-
--- +goose Down
-DROP TABLE header_sync_transactions;

db/migrations/00027_create_header_sync_receipts_table.sql

@@ -1,19 +0,0 @@
--- +goose Up
-CREATE TABLE header_sync_receipts
-(
-    id                  SERIAL PRIMARY KEY,
-    transaction_id      INTEGER NOT NULL REFERENCES header_sync_transactions (id) ON DELETE CASCADE,
-    header_id           INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
-    contract_address_id INTEGER NOT NULL REFERENCES addresses (id) ON DELETE CASCADE,
-    cumulative_gas_used NUMERIC,
-    gas_used            NUMERIC,
-    state_root          VARCHAR(66),
-    status              INTEGER,
-    tx_hash             VARCHAR(66),
-    rlp                 BYTEA,
-    UNIQUE (header_id, transaction_id)
-);
-
-
--- +goose Down
-DROP TABLE header_sync_receipts;

db/migrations/00028_create_uncles_table.sql

@@ -1,16 +0,0 @@
--- +goose Up
-CREATE TABLE public.uncles (
-  id                    SERIAL PRIMARY KEY,
-  hash                  VARCHAR(66) NOT NULL,
-  block_id              INTEGER NOT NULL REFERENCES blocks (id) ON DELETE CASCADE,
-  reward                NUMERIC NOT NULL,
-  miner                 VARCHAR(42) NOT NULL,
-  raw                   JSONB,
-  block_timestamp       NUMERIC,
-  eth_node_id           INTEGER NOT NULL REFERENCES eth_nodes (id) ON DELETE CASCADE,
-  eth_node_fingerprint  VARCHAR(128),
-  UNIQUE (block_id, hash)
-);
-
--- +goose Down
-DROP TABLE public.uncles;

db/migrations/00029_create_header_sync_logs_table.sql

@@ -1,22 +0,0 @@
--- +goose Up
--- SQL in this section is executed when the migration is applied.
-CREATE TABLE header_sync_logs
-(
-    id           SERIAL PRIMARY KEY,
-    header_id    INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
-    address      INTEGER NOT NULL REFERENCES addresses (id) ON DELETE CASCADE,
-    topics       BYTEA[],
-    data         BYTEA,
-    block_number BIGINT,
-    block_hash   VARCHAR(66),
-    tx_hash      VARCHAR(66),
-    tx_index     INTEGER,
-    log_index    INTEGER,
-    raw          JSONB,
-    transformed  BOOL    NOT NULL DEFAULT FALSE,
-    UNIQUE (header_id, tx_index, log_index)
-);
-
--- +goose Down
--- SQL in this section is executed when the migration is rolled back.
-DROP TABLE header_sync_logs;

db/migrations/00030_create_watched_logs_table.sql

@@ -1,12 +0,0 @@
--- +goose Up
--- SQL in this section is executed when the migration is applied.
-CREATE TABLE public.watched_logs
-(
-    id               SERIAL PRIMARY KEY,
-    contract_address VARCHAR(42),
-    topic_zero       VARCHAR(66)
-);
-
--- +goose Down
--- SQL in this section is executed when the migration is rolled back.
-DROP TABLE public.watched_logs;

dockerfiles/migrations/Dockerfile

@@ -0,0 +1,40 @@
+FROM golang:alpine
+
+RUN apk --update --no-cache add make git g++ linux-headers
+# DEBUG
+RUN apk add busybox-extras
+
+# this is probably a noob move, but I want apk from alpine for the above but need to avoid Go 1.13 below as this error still occurs https://github.com/ipfs/go-ipfs/issues/6603
+FROM golang:1.12.4 as builder
+
+# Get and build vulcanizedb
+ADD . /go/src/github.com/vulcanize/ipfs-chain-watcher
+
+# Build migration tool
+RUN go get -u -d github.com/pressly/goose/cmd/goose
+WORKDIR /go/src/github.com/pressly/goose/cmd/goose
+RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose .
+
+WORKDIR /go/src/github.com/vulcanize/ipfs-chain-watcher
+
+# app container
+FROM alpine
+
+ARG USER
+
+RUN adduser -Du 5000 $USER
+WORKDIR /app
+RUN chown $USER /app
+USER $USER
+
+# chown first so dir is writable
+# note: using $USER is merged, but not in the stable release yet
+COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/dockerfiles/migrations/startup_script.sh .
+
+
+# keep binaries immutable
+COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
+COPY --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/db/migrations migrations/vulcanizedb
+# XXX dir is already writeable RUN touch vulcanizedb.log
+
+CMD ["./startup_script.sh"]

dockerfiles/migrations/startup_script.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+# Runs the db migrations and starts the super node services
+
+# Exit if the variable tests fail
+set -e
+set +x
+
+# Check the database variables are set
+test $DATABASE_HOSTNAME
+test $DATABASE_NAME
+test $DATABASE_PORT
+test $DATABASE_USER
+test $DATABASE_PASSWORD
+set +e
+
+# Construct the connection string for postgres
+VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
+
+# Run the DB migrations
+echo "Connecting with: $VDB_PG_CONNECT"
+echo "Running database migrations"
+./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
+
+
+# If the db migrations ran without err
+if [[ $? -eq 0 ]]; then
+    echo "Migrations ran successfully"
+    exit 0
+else
+    echo "Could not run migrations. Are the database details correct?"
+    exit 1
+fi

dockerfiles/postgraphile/Dockerfile

@@ -0,0 +1,8 @@
+FROM node:alpine
+
+RUN npm install -g postgraphile
+RUN npm install -g postgraphile-plugin-connection-filter
+RUN npm install -g @graphile/pg-pubsub
+
+EXPOSE 5000
+ENTRYPOINT ["postgraphile"]

dockerfiles/postgraphile/docker-compose.yml

@@ -0,0 +1,60 @@
+version: '3.2'
+
+services:
+  db:
+    restart: always
+    image: postgres:10.12-alpine
+    environment:
+      POSTGRES_USER: "vdbm"
+      POSTGRES_DB: "vulcanize_public"
+      POSTGRES_PASSWORD: "password"
+    volumes:
+     - vulcanizedb_db_data:/var/lib/postgresql/data
+    expose:
+     - "5432"
+    ports:
+     - "127.0.0.1:8079:5432"
+
+  migrations:
+    restart: on-failure
+    depends_on:
+     - db
+    build:
+      context: ./../../
+      cache_from:
+        - alpine:latest
+      dockerfile: ./dockerfiles/migrations/Dockerfile
+      args:
+        USER: "vdbm"
+    environment:
+      DATABASE_NAME: "vulcanize_public"
+      DATABASE_HOSTNAME: "db"
+      DATABASE_PORT: 5432
+      DATABASE_USER: "vdbm"
+      DATABASE_PASSWORD: "password"
+
+  graphql:
+    restart: always
+    depends_on:
+     - db
+     - migrations
+    build:
+      context: ./../../
+      cache_from:
+       - node:alpine
+      dockerfile: ./dockerfiles/postgraphile/Dockerfile
+    expose:
+     - "5000"
+    ports:
+     - "127.0.0.1:5000:5000"
+    command: ["--plugins", "@graphile/pg-pubsub",
+              "--subscriptions",
+              "--simple-subscriptions",
+              "--connection", "postgres://vdbm:password@db:5432/vulcanize_public",
+              "--port", "5000",
+              "-n", "0.0.0.0",
+              "--schema", "public,btc,eth",
+              "--append-plugins", "postgraphile-plugin-connection-filter"]
+
+volumes:
+  vulcanizedb_db_data:

dockerfiles/rinkeby/Dockerfile

@@ -1,9 +0,0 @@
-FROM golang:1.10.3-alpine3.7
-
-RUN apk add --no-cache make gcc musl-dev
-
-ADD . /go/src/github.com/vulcanize/vulcanizedb
-WORKDIR /go/src/github.com/vulcanize/vulcanizedb
-RUN go build -o /app main.go
-
-ENTRYPOINT ["/app"]

dockerfiles/rinkeby/config.toml

@@ -1,9 +0,0 @@
-[database]
-name = "vulcanizedb"
-hostname = "postgres"
-port = 5432
-user = "postgres"
-password = "postgres"
-
-[client]
-ipcPath = "/geth/geth.ipc"

dockerfiles/rinkeby/docker-compose.yml

@@ -1,63 +0,0 @@
-version: '2.2'
-
-services:
-
-  vulcanizedb:
-    build:
-      context: ./../../
-      dockerfile: dockerfiles/rinkeby/Dockerfile
-    container_name: rinkeby_vulcanizedb
-    command: "sync --starting-block-number 0 --config /config.toml"
-    volumes:
-      - "./config.toml:/config.toml"
-      - "vulcanizedb_geth_data:/geth"
-    networks:
-      vulcanizedb_net:
-
-  migrations:
-    image: migrate/migrate:v3.3.0
-    container_name: rinkeby_vulcanizedb_migrations
-    depends_on:
-      postgres:
-        condition: service_healthy
-    command: -database postgresql://postgres:postgres@postgres:5432/vulcanizedb?sslmode=disable -path /migrations up
-    volumes:
-      - ./../../db/migrations:/migrations
-    networks:
-      vulcanizedb_net:
-
-  postgres:
-    image: postgres:9.6.5-alpine
-    container_name: rinkeby_vulcanizedb_postgres
-    environment:
-      POSTGRES_USER: postgres
-      POSTGRES_DB: vulcanizedb
-      POSTGRES_PASSWORD: postgres
-    volumes:
-      - "vulcanizedb_db_data:/var/lib/postgresql/data"
-    networks:
-      vulcanizedb_net:
-    healthcheck:
-      test: ["CMD", "pg_isready"]
-      interval: 5s
-      timeout: 5s
-      retries: 30
-  
-  geth:
-    image: ethereum/client-go:v1.8.11
-    container_name: rinkeby_vulcanizedb_geth
-    cpus: 0.3
-    hostname: eth
-    command: '--rinkeby --rpc --rpcaddr="0.0.0.0" --rpcvhosts="geth"'
-    volumes:
-      - "vulcanizedb_geth_data:/root/.ethereum/rinkeby"
-    networks:
-      vulcanizedb_net:
-
-volumes:
-  vulcanizedb_geth_data:
-  vulcanizedb_db_data:
-
-networks:
-  vulcanizedb_net:
-    driver: bridge

dockerfiles/super_node/Dockerfile

@@ -0,0 +1,48 @@
+FROM golang:1.12-alpine as builder
+
+RUN apk --update --no-cache add make git g++ linux-headers
+# DEBUG
+RUN apk add busybox-extras
+
+# Get and build vulcanizedb
+ADD . /go/src/github.com/vulcanize/ipfs-chain-watcher
+WORKDIR /go/src/github.com/vulcanize/ipfs-chain-watcher
+RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o vulcanizedb .
+
+# Build migration tool
+RUN go get -u -d github.com/pressly/goose/cmd/goose
+WORKDIR /go/src/github.com/pressly/goose/cmd/goose
+RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose .
+
+WORKDIR /go/src/github.com/vulcanize/ipfs-chain-watcher
+
+# app container
+FROM alpine
+
+ARG USER
+ARG CONFIG_FILE
+ARG EXPOSE_PORT_1
+ARG EXPOSE_PORT_2
+
+RUN adduser -Du 5000 $USER
+WORKDIR /app
+RUN chown $USER /app
+USER $USER
+
+# chown first so dir is writable
+# note: using $USER is merged, but not in the stable release yet
+COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/$CONFIG_FILE config.toml
+COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/dockerfiles/super_node/startup_script.sh .
+COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/dockerfiles/super_node/entrypoint.sh .
+
+
+# keep binaries immutable
+COPY --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/vulcanizedb vulcanizedb
+COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
+COPY --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/db/migrations migrations/vulcanizedb
+COPY --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/environments environments
+
+EXPOSE $EXPOSE_PORT_1
+EXPOSE $EXPOSE_PORT_2
+
+ENTRYPOINT ["/app/entrypoint.sh"]

dockerfiles/super_node/docker-compose.yml

@@ -0,0 +1,87 @@
+version: '3.2'
+
+services:
+  db:
+    restart: always
+    image: postgres:10.12-alpine
+    environment:
+      POSTGRES_USER: "vdbm"
+      POSTGRES_DB: "vulcanize_public"
+      POSTGRES_PASSWORD: "password"
+    volumes:
+     - vulcanizedb_db_data:/var/lib/postgresql/data
+    expose:
+     - "5432"
+    ports:
+     - "127.0.0.1:8079:5432"
+
+  btc:
+    depends_on:
+      - db
+    build:
+      context: ./../../
+      cache_from:
+       - alpine:latest
+       - golang:1.12.4
+      dockerfile: ./dockerfiles/super_node/Dockerfile
+      args:
+        USER: "vdbm"
+        CONFIG_FILE: ./environments/superNodeBTC.toml
+    environment:
+      VDB_COMMAND: "superNode"
+      DATABASE_NAME: "vulcanize_public"
+      DATABASE_HOSTNAME: "db"
+      DATABASE_PORT: 5432
+      DATABASE_USER: "vdbm"
+      DATABASE_PASSWORD: "password"
+    ports:
+     - "127.0.0.1:8082:8082"
+     - "127.0.0.1:8083:8083"
+
+  eth:
+    depends_on:
+      - db
+    build:
+      context: ./../../
+      cache_from:
+       - alpine:latest
+       - golang:1.12.4
+      dockerfile: ./dockerfiles/super_node/Dockerfile
+      args:
+        USER: "vdbm"
+        CONFIG_FILE: ./environments/superNodeETH.toml
+    environment:
+      VDB_COMMAND: "superNode"
+      DATABASE_NAME: "vulcanize_public"
+      DATABASE_HOSTNAME: "db"
+      DATABASE_PORT: 5432
+      DATABASE_USER: "vdbm"
+      DATABASE_PASSWORD: "password"
+    ports:
+     - "127.0.0.1:8080:8080"
+     - "127.0.0.1:8081:8081"
+
+  graphql:
+    restart: always
+    depends_on:
+      - db
+    build:
+      context: ./../../
+      cache_from:
+      - node:alpine
+      dockerfile: ./dockerfiles/postgraphile/Dockerfile
+    expose:
+      - "5000"
+    ports:
+      - "127.0.0.1:5000:5000"
+    command: ["--plugins", "@graphile/pg-pubsub",
+              "--subscriptions",
+              "--simple-subscriptions",
+              "--connection", "postgres://vdbm:password@db:5432/vulcanize_public",
+              "--port", "5000",
+              "-n", "0.0.0.0",
+              "--schema", "public,btc,eth",
+              "--append-plugins", "postgraphile-plugin-connection-filter"]
+
+volumes:
+  vulcanizedb_db_data:

dockerfiles/super_node/entrypoint.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+# Runs the db migrations and starts the super node services
+
+# Exit if the variable tests fail
+set -e
+set +x
+
+# Check the database variables are set
+# XXX set defaults, don't silently fail
+#test $DATABASE_HOSTNAME
+#test $DATABASE_NAME
+#test $DATABASE_PORT
+#test $DATABASE_USER
+#test $DATABASE_PASSWORD
+#test $IPFS_INIT
+#test $IPFS_PATH
+VDB_COMMAND=${VDB_COMMAND:-superNode}
+set +e
+
+# Construct the connection string for postgres
+VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
+
+# Run the DB migrations
+echo "Connecting with: $VDB_PG_CONNECT"
+echo "Running database migrations"
+./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
+rv=$?
+
+if [ $rv != 0 ]; then
+  echo "Could not run migrations. Are the database details correct?"
+  exit 1
+fi
+
+
+echo "Beginning the vulcanizedb process"
+VDB_CONFIG_FILE=${VDB_CONFIG_FILE:-config.toml}
+DEFAULT_OPTIONS="--config=$VDB_CONFIG_FILE"
+VDB_FULL_CL=${VDB_FULL_CL:-$VDB_COMMAND $DEFAULT_OPTIONS}
+echo running: ./vulcanizedb $VDB_FULL_CL $@
+
+case "$1" in
+  "/bin/sh" )
+    echo dropping to shell
+    exec /bin/sh
+esac
+
+vdb_args="$@"
+# default is to use the config passed by the build arg
+if [[ -z "$vdb_args" ]]; then
+  vdb_args="--config=config.toml"
+fi
+
+echo running: ./vulcanizedb $vdb_args
+./vulcanizedb $vdb_args
+rv=$?
+
+if [ $rv != 0 ]; then
+  echo "VulcanizeDB startup failed"
+  exit 1
+fi

dockerfiles/super_node/startup_script.sh

@@ -0,0 +1,65 @@
+#!/bin/sh
+# Runs the db migrations and starts the super node services
+
+# Exit if the variable tests fail
+set -e
+set +x
+
+# Check the database variables are set
+test $DATABASE_HOSTNAME
+test $DATABASE_NAME
+test $DATABASE_PORT
+test $DATABASE_USER
+test $DATABASE_PASSWORD
+test $IPFS_INIT
+test $IPFS_PATH
+test $VDB_COMMAND
+set +e
+
+# Export our database variables so that the IPFS Postgres plugin can use them
+export IPFS_PGHOST=$DATABASE_HOSTNAME
+export IPFS_PGUSER=$DATABASE_USER
+export IPFS_PGDATABASE=$DATABASE_NAME
+export IPFS_PGPORT=$DATABASE_PORT
+export IPFS_PGPASSWORD=$DATABASE_PASSWORD
+
+# Construct the connection string for postgres
+VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
+
+# Run the DB migrations
+echo "Connecting with: $VDB_PG_CONNECT"
+echo "Running database migrations"
+./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
+
+
+# If the db migrations ran without err
+if [[ $? -eq 0 ]]; then
+    # and IPFS_INIT is true
+    if [[ "$IPFS_INIT" = true ]] ; then
+        # initialize PG-IPFS
+        echo "Initializing Postgres-IPFS profile"
+        ./ipfs init --profile=postgresds
+    else
+        echo "IPFS profile already initialized, skipping initialization"
+    fi
+else
+    echo "Could not run migrations. Are the database details correct?"
+    exit 1
+fi
+
+# If IPFS initialization was successful
+if [[ $? -eq 0 ]]; then
+    echo "Running the VulcanizeDB process"
+    ./vulcanizedb ${VDB_COMMAND} --config=config.toml
+else
+    echo "Could not initialize IPFS."
+    exit 1
+fi
+
+# If VulcanizeDB process was successful
+if [ $? -eq 0 ]; then
+    echo "VulcanizeDB process ran successfully"
+else
+    echo "Could not start VulcanizeDB process. Is the config file correct?"
+    exit 1
+fi

documentation/apis.md

@@ -0,0 +1,262 @@
+## ipfs-blockchain-watcher APIs
+We can expose a number of different APIs for remote access to ipfs-blockchain-watcher data
+
+
+### Table of Contents
+1. [Postgraphile](#postgraphile)
+1. [RPC Subscription Interface](#rpc-subscription-interface)
+1. [Native API Recapitulation](#native-api-recapitulation)
+
+
+### Postgraphile
+ipfs-blockchain-watcher stores all processed data in Postgres using PG-IPFS, this includes all of the IPLD objects.
+[Postgraphile](https://www.graphile.org/postgraphile/) can be used to expose GraphQL endpoints for the Postgres tables.
+
+e.g. 
+
+`postgraphile --plugins @graphile/pg-pubsub --subscriptions --simple-subscriptions -c postgres://localhost:5432/vulcanize_public?sslmode=disable -s public,btc,eth -a -j`
+
+
+This will stand up a Postgraphile server on the public, eth, and btc schemas- exposing GraphQL endpoints for all of the tables contained under those schemas.
+All of their data can then be queried with standard [GraphQL](https://graphql.org) queries.
+
+
+### RPC Subscription Interface
+A direct, real-time subscription to the data being processed by ipfs-blockchain-watcher can be established over WS or IPC through the [Stream](../pkg/watcher/api.go#L53) RPC method.
+This method is not chain-specific and each chain-type supports it, it is accessed under the "vdb" namespace rather than a chain-specific namespace. An interface for
+subscribing to this endpoint is provided [here](../pkg/streamer/super_node_streamer.go).
+
+When subscribing to this endpoint, the subscriber provides a set of RLP-encoded subscription parameters. These parameters will be chain-specific, and are used
+by ipfs-blockchain-watcher to filter and return a requested subset of chain data to the subscriber. (e.g. [BTC](../pkg/btc/subscription_config.go), [ETH](../../pkg/eth/subscription_config.go)).
+
+#### Ethereum RPC Subscription
+An example of how to subscribe to a real-time Ethereum data feed from ipfs-blockchain-watcher using the `Stream` RPC method is provided below
+
+```go
+    package main 
+
+    import (
+    	"github.com/ethereum/go-ethereum/rlp"
+    	"github.com/ethereum/go-ethereum/rpc"
+    	"github.com/spf13/viper"
+    	
+        "github.com/vulcanize/ipfs-chain-watcher/pkg/client"
+        "github.com/vulcanize/ipfs-chain-watcher/pkg/eth"
+        "github.com/vulcanize/ipfs-chain-watcher/pkg/streamer"
+        "github.com/vulcanize/ipfs-chain-watcher/pkg/watcher"
+    )
+
+    config, _ := eth.NewEthSubscriptionConfig()
+    rlpConfig, _ := rlp.EncodeToBytes(config)
+    vulcPath := viper.GetString("superNode.ethSubscription.path")
+    rawRPCClient, _ := rpc.Dial(vulcPath)
+    rpcClient := client.NewRPCClient(rawRPCClient, vulcPath)
+    stream := streamer.NewSuperNodeStreamer(rpcClient)
+    payloadChan := make(chan watcher.SubscriptionPayload, 20000)
+    subscription, _ := stream.Stream(payloadChan, rlpConfig)
+    for {
+        select {
+        case payload := <- payloadChan:
+            // do something with the subscription payload
+        case err := <- subscription.Err():
+            // do something with the subscription error
+        }
+    }
+```
+
+The .toml file being used to fill the Ethereum subscription config would look something like this:
+
+```toml
+[superNode]
+    [superNode.ethSubscription]
+        historicalData = false
+        historicalDataOnly = false
+        startingBlock = 0
+        endingBlock = 0
+        wsPath = "ws://127.0.0.1:8080"
+        [superNode.ethSubscription.headerFilter]
+            off = false
+            uncles = false
+        [superNode.ethSubscription.txFilter]
+            off = false
+            src = []
+            dst = []
+        [superNode.ethSubscription.receiptFilter]
+            off = false
+            contracts = []
+            topic0s = []
+            topic1s = []
+            topic2s = []
+            topic3s = []
+        [superNode.ethSubscription.stateFilter]
+            off = false
+            addresses = []
+           intermediateNodes = false
+        [superNode.ethSubscription.storageFilter]
+            off = true
+            addresses = []
+            storageKeys = []
+            intermediateNodes = false
+```
+
+These configuration parameters are broken down as follows:
+
+`ethSubscription.wsPath` is used to define the SuperNode ws url OR ipc endpoint we subscribe to
+
+`ethSubscription.historicalData` specifies whether or not ipfs-blockchain-watcher should look up historical data in its cache and
+send that to the subscriber, if this is set to `false` then we only streams newly synced/incoming data
+
+`ethSubscription.historicalDataOnly` will tell ipfs-blockchain-watcher to only send historical data with the specified range and
+not stream forward syncing data
+
+`ethSubscription.startingBlock` is the starting block number for the range we want to receive data in
+
+`ethSubscription.endingBlock` is the ending block number for the range we want to receive data in;
+setting to 0 means there is no end/we will continue streaming indefinitely.
+
+`ethSubscription.headerFilter` has two sub-options: `off` and `uncles`. 
+
+- Setting `off` to true tells ipfs-blockchain-watcher to not send any headers to the subscriber
+- setting `uncles` to true tells ipfs-blockchain-watcher to send uncles in addition to normal headers.
+
+`ethSubscription.txFilter` has three sub-options: `off`, `src`, and `dst`. 
+
+- Setting `off` to true tells ipfs-blockchain-watcher to not send any transactions to the subscriber
+- `src` and `dst` are string arrays which can be filled with ETH addresses we want to filter transactions for,
+if they have any addresses then ipfs-blockchain-watcher will only send transactions that were sent or received by the addresses contained
+in `src` and `dst`, respectively.
+
+`ethSubscription.receiptFilter` has four sub-options: `off`, `topics`, `contracts` and `matchTxs`. 
+
+- Setting `off` to true tells ipfs-blockchain-watcher to not send any receipts to the subscriber
+- `topic0s` is a string array which can be filled with event topics we want to filter for,
+if it has any topics then ipfs-blockchain-watcher will only send receipts that contain logs which have that topic0.
+- `contracts` is a string array which can be filled with contract addresses we want to filter for, if it contains any contract addresses the super node will
+only send receipts that correspond to one of those contracts. 
+- `matchTrxs` is a bool which when set to true any receipts that correspond to filtered for transactions will be sent by the super node, regardless of whether or not the receipt satisfies the `topics` or `contracts` filters.
+
+`ethSubscription.stateFilter` has three sub-options: `off`, `addresses`, and `intermediateNodes`. 
+
+- Setting `off` to true tells ipfs-blockchain-watcher to not send any state data to the subscriber
+- `addresses` is a string array which can be filled with ETH addresses we want to filter state for,
+if it has any addresses then ipfs-blockchain-watcher will only send state leafs (accounts) corresponding to those account addresses. 
+- By default ipfs-blockchain-watcher only sends along state leafs, if we want to receive branch and extension nodes as well `intermediateNodes` can be set to `true`.
+
+`ethSubscription.storageFilter` has four sub-options: `off`, `addresses`, `storageKeys`, and `intermediateNodes`. 
+
+- Setting `off` to true tells ipfs-blockchain-watcher to not send any storage data to the subscriber
+- `addresses` is a string array which can be filled with ETH addresses we want to filter storage for,
+if it has any addresses then ipfs-blockchain-watcher will only send storage nodes from the storage tries at those state addresses.
+- `storageKeys` is another string array that can be filled with storage keys we want to filter storage data for. It is important to note that the storage keys need to be the actual keccak256 hashes, whereas
+the addresses in the `addresses` fields are pre-hashed ETH addresses.
+- By default ipfs-blockchain-watcher only sends along storage leafs, if we want to receive branch and extension nodes as well `intermediateNodes` can be set to `true`.
+
+### Bitcoin RPC Subscription:
+An example of how to subscribe to a real-time Bitcoin data feed from ipfs-blockchain-watcher using the `Stream` RPC method is provided below
+
+```go
+    package main 
+
+    import (
+    	"github.com/ethereum/go-ethereum/rlp"
+    	"github.com/ethereum/go-ethereum/rpc"
+    	"github.com/spf13/viper"
+    	
+    	"github.com/vulcanize/ipfs-chain-watcher/libraries/shared/streamer"
+    	"github.com/vulcanize/ipfs-chain-watcher/pkg/eth/client"
+    	"github.com/vulcanize/ipfs-chain-watcher/pkg/super_node"
+    	"github.com/vulcanize/ipfs-chain-watcher/pkg/super_node/btc"
+    )
+
+    config, _ := btc.NewBtcSubscriptionConfig()
+    rlpConfig, _ := rlp.EncodeToBytes(config)
+    vulcPath := viper.GetString("superNode.btcSubscription.path")
+    rawRPCClient, _ := rpc.Dial(vulcPath)
+    rpcClient := client.NewRPCClient(rawRPCClient, vulcPath)
+    stream := streamer.NewSuperNodeStreamer(rpcClient)
+    payloadChan := make(chan super_node.SubscriptionPayload, 20000)
+    subscription, _ := stream.Stream(payloadChan, rlpConfig)
+    for {
+        select {
+        case payload := <- payloadChan:
+            // do something with the subscription payload
+        case err := <- subscription.Err():
+            // do something with the subscription error
+        }
+    }
+```
+
+The .toml file being used to fill the Bitcoin subscription config would look something like this:
+
+```toml
+[superNode]
+    [superNode.btcSubscription]
+        historicalData = false
+        historicalDataOnly = false
+        startingBlock = 0
+        endingBlock = 0
+        wsPath = "ws://127.0.0.1:8080"
+        [superNode.btcSubscription.headerFilter]
+            off = false
+        [superNode.btcSubscription.txFilter]
+            off = false
+            segwit = false
+            witnessHashes = []
+            indexes = []
+            pkScriptClass = []
+            multiSig = false
+            addresses = []
+```
+
+These configuration parameters are broken down as follows:
+
+`btcSubscription.wsPath` is used to define the SuperNode ws url OR ipc endpoint we subscribe to
+
+`btcSubscription.historicalData` specifies whether or not ipfs-blockchain-watcher should look up historical data in its cache and
+send that to the subscriber, if this is set to `false` then ipfs-blockchain-watcher only streams newly synced/incoming data
+
+`btcSubscription.historicalDataOnly` will tell ipfs-blockchain-watcher to only send historical data with the specified range and
+not stream forward syncing data
+
+`btcSubscription.startingBlock` is the starting block number for the range we want to receive data in
+
+`btcSubscription.endingBlock` is the ending block number for the range we want to receive data in;
+setting to 0 means there is no end/we will continue streaming indefinitely.
+
+`btcSubscription.headerFilter` has one sub-option: `off`. 
+
+- Setting `off` to true tells ipfs-blockchain-watcher to
+not send any headers to the subscriber.
+- Additional header-filtering options will be added in the future.
+
+`btcSubscription.txFilter` has seven sub-options: `off`, `segwit`, `witnessHashes`, `indexes`, `pkScriptClass`, `multiSig`, and `addresses`.
+
+- Setting `off` to true tells ipfs-blockchain-watcher to not send any transactions to the subscriber.
+- Setting `segwit` to true tells ipfs-blockchain-watcher to only send segwit transactions.
+- `witnessHashes` is a string array that can be filled with witness hash string; if it contains any hashes ipfs-blockchain-watcher will only send transactions that contain one of those hashes.
+- `indexes` is an int64 array that can be filled with tx index numbers; if it contains any integers ipfs-blockchain-watcher will only send transactions at those indexes (e.g. `[0]` will send only coinbase transactions)
+- `pkScriptClass` is an uint8 array that can be filled with pk script class numbers; if it contains any integers ipfs-blockchain-watcher will only send transactions that have at least one tx output with one of the specified pkscript classes;
+possible class types are 0 through 8 as defined [here](https://github.com/btcsuite/btcd/blob/master/txscript/standard.go#L52).
+- Setting `multisig` to true tells ipfs-blockchain-watcher to send only multi-sig transactions- to send only transaction that have at least one tx output that requires more than one signature to spend.
+- `addresses` is a string array that can be filled with btc address strings; if it contains any addresses ipfs-blockchain-watcher will only send transactions that have at least one tx output with at least one of the provided addresses.
+
+
+### Native API Recapitulation:
+In addition to providing novel Postgraphile and RPC-Subscription endpoints, we are working towards complete recapitulation of the
+standard chain APIs. This will allow direct compatibility with software that already makes use of the standard interfaces.
+
+#### Ethereum JSON-RPC API
+ipfs-blockchain-watcher currently faithfully recapitulates portions of the Ethereum JSON-RPC api standard.
+
+The currently supported endpoints include:  
+`eth_blockNumber`  
+`eth_getLogs`  
+`eth_getHeaderByNumber`  
+`eth_getBlockByNumber`  
+`eth_getBlockByHash`  
+`eth_getTransactionByHash`  
+
+Additional endpoints will be added in the near future, with the immediate goal of recapitulating the largest set of "eth_" endpoints which can be provided as a service.
+
+#### Bitcoin JSON-RPC API:
+In the near future, the standard Bitcoin JSON-RPC interfaces will be implemented.

documentation/architecture.md

@@ -0,0 +1,136 @@
+# ipfs-blockchain-watcher architecture
+1. [Processes](#processes)
+1. [Command](#command)
+1. [Configuration](#config)
+1. [Database](#database)
+1. [APIs](#apis)
+1. [Resync](#resync)
+1. [IPFS Considerations](#ipfs-considerations)
+
+## Processes
+ipfs-blockchain-watcher is a [service](../pkg/super_node/service.go#L61) comprised of the following interfaces:
+
+* [Payload Fetcher](../pkg/super_node/shared/interfaces.go#L29): Fetches raw chain data from a half-duplex endpoint (HTTP/IPC), used for historical data fetching. ([BTC](../../pkg/super_node/btc/payload_fetcher.go), [ETH](../../pkg/super_node/eth/payload_fetcher.go)).
+* [Payload Streamer](../pkg/super_node/shared/interfaces.go#L24): Streams raw chain data from a full-duplex endpoint (WebSocket/IPC), used for syncing data at the head of the chain in real-time. ([BTC](../../pkg/super_node/btc/http_streamer.go), [ETH](../../pkg/super_node/eth/streamer.go)).
+* [Payload Converter](../pkg/super_node/shared/interfaces.go#L34): Converters raw chain data to an intermediary form prepared for IPFS publishing. ([BTC](../../pkg/super_node/btc/converter.go), [ETH](../../pkg/super_node/eth/converter.go)).
+* [IPLD Publisher](../pkg/super_node/shared/interfaces.go#L39): Publishes the converted data to IPFS, returning their CIDs and associated metadata for indexing. ([BTC](../../pkg/super_node/btc/publisher.go), [ETH](../../pkg/super_node/eth/publisher.go)).
+* [CID Indexer](../pkg/super_node/shared/interfaces.go#L44): Indexes CIDs in Postgres with their associated metadata. This metadata is chain specific and selected based on utility. ([BTC](../../pkg/super_node/btc/indexer.go), [ETH](../../pkg/super_node/eth/indexer.go)).
+* [CID Retriever](../pkg/super_node/shared/interfaces.go#L54): Retrieves CIDs from Postgres by searching against their associated metadata, is used to lookup data to serve API requests/subscriptions. ([BTC](../../pkg/super_node/btc/retriever.go), [ETH](../../pkg/super_node/eth/retriever.go)).
+* [IPLD Fetcher](../pkg/super_node/shared/interfaces.go#L62): Fetches the IPLDs needed to service API requests/subscriptions from IPFS using retrieved CIDS; can route through a IPFS block-exchange to search for objects that are not directly available. ([BTC](../../pkg/super_node/btc/ipld_fetcher.go), [ETH](../../pkg/super_node/eth/ipld_fetcher.go))
+* [Response Filterer](../pkg/super_node/shared/interfaces.go#L49): Filters converted data payloads served to API subscriptions; filters according to the subscriber provided parameters. ([BTC](../../pkg/super_node/btc/filterer.go), [ETH](../../pkg/super_node/eth/filterer.go)).
+* [API](https://github.com/ethereum/go-ethereum/blob/master/rpc/types.go#L31): Expose RPC methods for clients to interface with the data. Chain-specific APIs should aim to recapitulate as much of the native API as possible. ([VDB](../../pkg/super_node/api.go), [ETH](../../pkg/super_node/eth/api.go)).
+
+
+Appropriating the service for a new chain is done by creating underlying types to satisfy these interfaces for
+the specifics of that chain.
+
+The service uses these interfaces to operate in any combination of three modes: `sync`, `serve`, and `backfill`.
+* Sync: Streams raw chain data at the head, converts and publishes it to IPFS, and indexes the resulting set of CIDs in Postgres with useful metadata.
+* BackFill: Automatically searches for and detects gaps in the DB; fetches, converts, publishes, and indexes the data to fill these gaps.
+* Serve: Opens up IPC, HTTP, and WebSocket servers on top of the ipfs-blockchain-watcher DB and any concurrent sync and/or backfill processes.
+
+
+These three modes are all operated through a single vulcanizeDB command: `watch`
+
+## Command
+
+Usage: `./ipfs-blockchain-watcher watch --config={config.toml}`
+
+Configuration can also be done through CLI options and/or environmental variables.
+CLI options can be found using `./ipfs-blockchain-watcher watch --help`.
+
+## Config
+
+Below is the set of universal config parameters for the ipfs-blockchain-watcher command, in .toml form, with the respective environmental variables commented to the side.
+This set of parameters needs to be set no matter the chain type.
+
+```toml
+[database]
+    name     = "vulcanize_public" # $DATABASE_NAME
+    hostname = "localhost" # $DATABASE_HOSTNAME
+    port     = 5432 # $DATABASE_PORT
+    user     = "vdbm" # $DATABASE_USER
+    password = "" # $DATABASE_PASSWORD
+
+[ipfs]
+    path = "~/.ipfs" # $IPFS_PATH
+    mode = "direct" # $IPFS_MODE
+
+[superNode]
+    chain = "bitcoin" # $SUPERNODE_CHAIN
+    server = true # $SUPERNODE_SERVER
+    ipcPath = "~/.vulcanize/vulcanize.ipc" # $SUPERNODE_IPC_PATH
+    wsPath = "127.0.0.1:8082" # $SUPERNODE_WS_PATH
+    httpPath = "127.0.0.1:8083" # $SUPERNODE_HTTP_PATH
+    sync = true # $SUPERNODE_SYNC
+    workers = 1 # $SUPERNODE_WORKERS
+    backFill = true # $SUPERNODE_BACKFILL
+    frequency = 45 # $SUPERNODE_FREQUENCY
+    batchSize = 1 # $SUPERNODE_BATCH_SIZE
+    batchNumber = 50 # $SUPERNODE_BATCH_NUMBER
+    timeout = 300 # $HTTP_TIMEOUT
+    validationLevel = 1 # $SUPERNODE_VALIDATION_LEVEL
+```
+
+Additional parameters need to be set depending on the specific chain.
+
+For Bitcoin:
+
+```toml
+[bitcoin]
+    wsPath  = "127.0.0.1:8332" # $BTC_WS_PATH
+    httpPath = "127.0.0.1:8332" # $BTC_HTTP_PATH
+    pass = "password" # $BTC_NODE_PASSWORD
+    user = "username" # $BTC_NODE_USER
+    nodeID = "ocd0" # $BTC_NODE_ID
+    clientName = "Omnicore" # $BTC_CLIENT_NAME
+    genesisBlock = "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f" # $BTC_GENESIS_BLOCK
+    networkID = "0xD9B4BEF9" # $BTC_NETWORK_ID
+```
+
+For Ethereum:
+
+```toml
+[ethereum]
+    wsPath  = "127.0.0.1:8546" # $ETH_WS_PATH
+    httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH
+    nodeID = "arch1" # $ETH_NODE_ID
+    clientName = "Geth" # $ETH_CLIENT_NAME
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK
+    networkID = "1" # $ETH_NETWORK_ID
+```
+
+## Database
+
+Currently, ipfs-blockchain-watcher persists all data to a single Postgres database. The migrations for this DB can be found [here](../../db/migrations).
+Chain-specific data is populated under a chain-specific schema (e.g. `eth` and `btc`) while shared data- such as the IPFS blocks table- is populated under the `public` schema.
+Subsequent watchers which act on the raw chain data should build and populate their own schemas or separate databases entirely.
+
+In the future, we will be moving to a foreign table based architecture wherein a single db is used for shared data while each watcher uses
+its own database and accesses and acts on the shared data through foreign tables. Isolating watchers to their own databases will prevent complications and
+conflicts between watcher db migrations.
+
+
+## APIs
+
+ipfs-blockchain-watcher provides mutliple types of APIs by which to interface with its data.
+More detailed information on the APIs can be found [here](apis.md).
+
+## Resync
+
+A separate command `resync` is available for directing the resyncing of data within specified ranges.
+This is useful if we want to re-validate a range of data using a new source or clean out bad/deprecated data.
+More detailed information on this command can be found [here](resync.md).
+
+## IPFS Considerations
+
+Currently the IPLD Publisher and Fetcher can either use internalized IPFS processes which interface with a local IPFS repository, or can interface
+directly with the backing Postgres database.
+Both these options circumvent the need to run a full IPFS daemon with a [go-ipld-eth](https://github.com/ipfs/go-ipld-eth) or [go-ipld-btc](https://github.com/ipld/go-ipld-btc) plugin.
+The former approach can lead to issues with lock-contention on the IPFS repo if another IPFS process is configured and running at the same $IPFS_PATH, it also necessitates the need for
+a locally configured IPFS repository. The later bypasses the need for a configured IPFS repository/$IPFS_PATH and allows all Postgres write operations at a given block height
+to occur in a single transaction, the only disadvantage is that by avoiding moving through an IPFS node intermediary we lose the direct ability to reach out to the block
+exchange for data we do not have locally.
+
+Once go-ipld-eth and go-ipld-btc have been updated to work with a modern version of PG-IPFS, an additional option will be provided to direct
+all publishing and fetching of IPLD objects through a remote IPFS daemon.

documentation/contributing.md

@@ -1,50 +0,0 @@
-# Contribution guidelines
-
-Contributions are welcome! Please open an Issues or Pull Request for any changes.
-
-In addition to core contributions, developers are encouraged to build their own custom transformers which
-can be run together with other custom transformers using the [composeAndExecute](../../staging/documentation/custom-transformers.md) command.
-
-## Pull Requests
-- `go fmt` is run as part of `make test` and `make integrationtest`, please make sure to check in the format changes.
-- Ensure that new code is well tested, including integration testing if applicable.
-- Make sure the build is passing.
-- Update the README or any [documentation files](./) as necessary. If editing the Readme, please
-conform to the
-[standard-readme specification](https://github.com/RichardLitt/standard-readme).
-- Once a Pull Request has received two approvals it can be merged in by a core developer.
-
-## Creating a new migration file
-1. `make new_migration NAME=add_columnA_to_table1`
-    - This will create a new timestamped migration file in `db/migrations`
-1. Write the migration code in the created file, under the respective `goose` pragma
-    - Goose automatically runs each migration in a transaction; don't add `BEGIN` and `COMMIT` statements.
-1. Core migrations should be committed in their `goose fix`ed form. To do this, run `make version_migrations` which
-converts timestamped migrations to migrations versioned by an incremented integer.
-
-## Diagrams
-- Diagrams were created with [draw.io](https://www.draw.io).
-- To update a diagram:
-  1. Go to [draw.io](https://www.draw.io).
-  1. Click on *File > Open from* and choose the location of the diagram you want to update.
-  1. Once open in draw.io, you may update it.
-  1. Export the diagram to this repository's directory and commit it.
-
-
-## Generating the Changelog
-We use [github-changelog-generator](https://github.com/github-changelog-generator/github-changelog-generator) to generate release Changelogs. To be consistent with previous Changelogs, the following flags should be passed to the command:
-
-```
---user vulcanize
---project vulcanizedb
---token {YOUR_GITHUB_TOKEN}
---no-issues
---usernames-as-github-logins
---since-tag {PREVIOUS_RELEASE_TAG}
-```
-
-For more information on why your github token is needed, and how to generate it see [https://github
-.com/github-changelog-generator/github-changelog-generator#github-token](https://github.com/github-changelog-generator/github-changelog-generator#github-token).
-
-## Code of Conduct
-VulcanizeDB follows the [Contributor Covenant Code of Conduct](https://www.contributor-covenant.org/version/1/4/code-of-conduct).

documentation/custom-transformers.md

@@ -1,196 +0,0 @@
-# Custom Transformers
-When the capabilities of the generic `contractWatcher` are not sufficient, custom transformers tailored to a specific
-purpose can be leveraged.
-
-Individual custom transformers can be composed together from any number of external repositories and executed as a
-single process using the `compose` and `execute` commands or the `composeAndExecute` command. This is accomplished by
-generating a Go plugin which allows the `vulcanizedb` binary to link to the external transformers, so long as they
-abide by one of the standard [interfaces](../staging/libraries/shared/transformer).
-
-## Writing custom transformers
-For help with writing different types of custom transformers please see below:
-
-Storage Transformers: transform data derived from contract storage tries
-   * [Guide](../../staging/libraries/shared/factories/storage/README.md)
-   * [Example](../../staging/libraries/shared/factories/storage/EXAMPLE.md)
-
-Event Transformers: transform data derived from Ethereum log events
-   * [Guide](../../staging/libraries/shared/factories/event/README.md)
-   * [Example 1](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registar)
-   * [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registry)
-   * [Example 3](https://github.com/vulcanize/ens_transformers/tree/master/transformers/resolver)
-
-Contract Transformers: transform data derived from Ethereum log events and use it to poll public contract methods
-   * [Example 1](https://github.com/vulcanize/account_transformers)
-   * [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/domain_records)
-
-## Preparing custom transformers to work as part of a plugin
-To plug in an external transformer we need to:
-
-1. Create a package that exports a variable `TransformerInitializer`, `StorageTransformerInitializer`, or `ContractTransformerInitializer` that are of type [TransformerInitializer](../staging/libraries/shared/transformer/event_transformer.go#L33)
-or [StorageTransformerInitializer](../../staging/libraries/shared/transformer/storage_transformer.go#L31),
-or [ContractTransformerInitializer](../../staging/libraries/shared/transformer/contract_transformer.go#L31), respectively
-2. Design the transformers to work in the context of their [event](../staging/libraries/shared/watcher/event_watcher.go#L83),
-[storage](../../staging/libraries/shared/watcher/storage_watcher.go#L53),
-or [contract](../../staging/libraries/shared/watcher/contract_watcher.go#L68) watcher execution modes
-3. Create db migrations to run against vulcanizeDB so that we can store the transformer output
-    * Do not `goose fix` the transformer migrations, this is to ensure they are always ran after the core vulcanizedb migrations which are kept in their fixed form
-    * Specify migration locations for each transformer in the config with the `exporter.transformer.migrations` fields
-    * If the base vDB migrations occupy this path as well, they need to be in their `goose fix`ed form
-    as they are [here](../../staging/db/migrations)
-
-To update a plugin repository with changes to the core vulcanizedb repository, use your dependency manager to install the desired version of vDB.
-
-## Building and Running Custom Transformers
-### Commands
-* The `compose`, `execute`, `composeAndExecute` commands require Go 1.11+ and use [Go plugins](https://golang
-.org/pkg/plugin/) which only work on Unix-based systems.
-
-* There is an ongoing [conflict](https://github.com/golang/go/issues/20481) between Go plugins and the use of vendored
-dependencies which imposes certain limitations on how the plugins are built.
-
-* Separate `compose` and `execute` commands allow pre-building and linking to the pre-built .so file. So, if
-these are run independently, instead of using `composeAndExecute`, a couple of things need to be considered:
-    * It is necessary that the .so file was built with the same exact dependencies that are present in the execution
-    environment, i.e. we need to `compose` and `execute` the plugin .so file with the same exact version of vulcanizeDB.
-    * The plugin migrations are run during the plugin's composition. As such, if `execute` is used to run a prebuilt .so
-    in a different environment than the one it was composed in, then the database structure will need to be loaded 
-    into the environment's Postgres database. This can either be done by manually loading the plugin's schema into 
-    Postgres, or by manually running the plugin's migrations.
-     
-* The `compose` and `composeAndExecute` commands assume you are in the vulcanizdb directory located at your system's 
-`$GOPATH`, and that the plugin dependencies are present at their `$GOPATH` directories.
-
-* The `execute` command does not require the plugin transformer dependencies be located in their `$GOPATH` directories,
-instead it expects a .so file (of the name specified in the config file) to be in
-`$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins/` and, as noted above, also expects the plugin db migrations to
- have already been ran against the database.
-
- * Usage:
-     * compose: `./vulcanizedb compose --config=environments/config_name.toml`
-
-     * execute: `./vulcanizedb execute --config=environments/config_name.toml`
-
-     * composeAndExecute: `./vulcanizedb composeAndExecute --config=environments/config_name.toml`
-
-### Flags
-The `execute` and `composeAndExecute` commands can be passed optional flags to specify the operation of the watchers:
-
-- `--recheck-headers`/`-r` - specifies whether to re-check headers for events after the header has already been queried for watched logs.
-Can be useful for redundancy if you suspect that your node is not always returning all desired logs on every query.
-Argument is expected to be a boolean: e.g. `-r=true`.
-Defaults to `false`.
-
-- `query-recheck-interval`/`-q` - specifies interval for re-checking storage diffs that haven been queued for later processing
-(by default, the storage watched queues storage diffs if transformer execution fails, on the assumption that subsequent data derived from the event transformers may enable us to decode storage keys that we don't recognize right now).
-Argument is expected to be a duration (integer measured in nanoseconds): e.g. `-q=10m30s` (for 10 minute, 30 second intervals).
-Defaults to `5m` (5 minutes).
-
-### Configuration
-A .toml config file is specified when executing the commands.
-The config provides information for composing a set of transformers from external repositories:
-
-```toml
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    user     = "vulcanize"
-    password = "vulcanize"
-    port     = 5432
-
-[client]
-    ipcPath  = "/Users/user/Library/Ethereum/geth.ipc"
-
-[exporter]
-    home     = "github.com/vulcanize/vulcanizedb"
-    name     = "exampleTransformerExporter"
-    save     = false
-    transformerNames = [
-        "transformer1",
-        "transformer2",
-        "transformer3",
-        "transformer4",
-    ]
-    [exporter.transformer1]
-        path = "path/to/transformer1"
-        type = "eth_event"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "0"
-    [exporter.transformer2]
-        path = "path/to/transformer2"
-        type = "eth_contract"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "0"
-    [exporter.transformer3]
-        path = "path/to/transformer3"
-        type = "eth_event"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "0"
-    [exporter.transformer4]
-        path = "path/to/transformer4"
-        type = "eth_storage"
-        repository = "github.com/account2/repo2"
-        migrations = "to/db/migrations"
-        rank = "1"
-```
-- `home` is the name of the package you are building the plugin for, in most cases this is github.com/vulcanize/vulcanizedb
-- `name` is the name used for the plugin files (.so and .go)   
-- `save` indicates whether or not the user wants to save the .go file instead of removing it after .so compilation. Sometimes useful for debugging/trouble-shooting purposes.
-- `transformerNames` is the list of the names of the transformers we are composing together, so we know how to access their submaps in the exporter map
-- `exporter.<transformerName>`s are the sub-mappings containing config info for the transformers
-    - `repository` is the path for the repository which contains the transformer and its `TransformerInitializer`
-    - `path` is the relative path from `repository` to the transformer's `TransformerInitializer` directory (initializer package).
-        - Transformer repositories need to be cloned into the user's $GOPATH (`go get`)
-    - `type` is the type of the transformer; indicating which type of watcher it works with (for now, there are only two options: `eth_event` and `eth_storage`)
-        - `eth_storage` indicates the transformer works with the [storage watcher](../../staging/libraries/shared/watcher/storage_watcher.go)
-         that fetches state and storage diffs from an ETH node (instead of, for example, from IPFS)
-        - `eth_event` indicates the transformer works with the [event watcher](../../staging/libraries/shared/watcher/event_watcher.go)
-         that fetches event logs from an ETH node
-        - `eth_contract` indicates the transformer works with the [contract watcher](../staging/libraries/shared/watcher/contract_watcher.go)
-        that is made to work with [contract_watcher pkg](../../staging/pkg/contract_watcher)
-        based transformers which work with either a header or full sync vDB to watch events and poll public methods ([example1](https://github.com/vulcanize/account_transformers/tree/master/transformers/account/light), [example2](https://github.com/vulcanize/ens_transformers/tree/working/transformers/domain_records))
-    - `migrations` is the relative path from `repository` to the db migrations directory for the transformer
-    - `rank` determines the order that migrations are ran, with lower ranked migrations running first
-        - this is to help isolate any potential conflicts between transformer migrations
-        - start at "0" 
-        - use strings
-        - don't leave gaps
-        - transformers with identical migrations/migration paths should share the same rank
-- Note: If any of the imported transformers need additional config variables those need to be included as well   
-
-This information is used to write and build a Go plugin which exports the configured transformers.
-These transformers are loaded onto their specified watchers and executed.
-
-Transformers of different types can be run together in the same command using a single config file or in separate instances using different config files   
-
-The general structure of a plugin .go file, and what we would see built with the above config is shown below
-
-```go
-package main
-
-import (
-	interface1 "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	transformer1 "github.com/account/repo/path/to/transformer1"
-	transformer2 "github.com/account/repo/path/to/transformer2"
-	transformer3 "github.com/account/repo/path/to/transformer3"
-	transformer4 "github.com/account2/repo2/path/to/transformer4"
-)
-
-type exporter string
-
-var Exporter exporter
-
-func (e exporter) Export() []interface1.EventTransformerInitializer, []interface1.StorageTransformerInitializer, []interface1.ContractTransformerInitializer {
-	return []interface1.TransformerInitializer{
-            transformer1.TransformerInitializer,
-            transformer3.TransformerInitializer,
-        },     []interface1.StorageTransformerInitializer{
-            transformer4.StorageTransformerInitializer,
-        },     []interface1.ContractTransformerInitializer{
-            transformer2.TransformerInitializer,
-        }
-}
-```

documentation/data-syncing.md

@@ -1,81 +0,0 @@
-# Syncing commands
-These commands are used to sync raw Ethereum data into Postgres, with varying levels of data granularity.
-
-## headerSync
-Syncs block headers from a running Ethereum node into the VulcanizeDB table `headers`.
-- Queries the Ethereum node using RPC calls.
-- Validates headers from the last 15 blocks to ensure that data is up to date.
-- Useful when you want a minimal baseline from which to track targeted data on the blockchain (e.g. individual smart
-contract storage values or event logs).
-- Handles chain reorgs by [validating the most recent blocks' hashes](../pkg/history/header_validator.go). If the hash is
-different from what we have already stored in the database, the header record will be updated.
-
-#### Usage
-- Run: `./vulcanizedb headerSync --config <config.toml> --starting-block-number <block-number>`
-- The config file must be formatted as follows, and should contain an ipc path to a running Ethereum node:
-```toml
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    user     = "vulcanize"
-    password = "vulcanize"
-    port     = 5432
-
-[client]
-    ipcPath  = <path to a running Ethereum node>
-```
-- Alternatively, the ipc path can be passed as a flag instead `--client-ipcPath`.
-
-## fullSync
-Syncs blocks, transactions, receipts and logs from a running Ethereum node into VulcanizeDB tables named
-`blocks`, `uncles`, `full_sync_transactions`, `full_sync_receipts` and `logs`. 
-- Queries the Ethereum node using RPC calls.
-- Validates headers from the last 15 blocks to ensure that data is up to date.
-- Useful when you want to maintain a broad cache of what's happening on the blockchain.
-- Handles chain reorgs by [validating the most recent blocks' hashes](../pkg/history/header_validator.go). If the hash is
-different from what we have already stored in the database, the header record will be updated.
-
-#### Usage
-- Run `./vulcanizedb fullSync --config <config.toml> --starting-block-number <block-number>`
-- The config file must be formatted as follows, and should contain an ipc path to a running Ethereum node:
-```toml
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    user     = "vulcanize"
-    password = "vulcanize"
-    port     = 5432
-
-[client]
-    ipcPath  = <path to a running Ethereum node>
-```
-- Alternatively, the ipc path can be passed as a flag instead `--client-ipcPath`.
-
-*Please note, that if you are fast syncing your Ethereum node, wait for the initial sync to finish.*
-
-## coldImport
-Syncs VulcanizeDB from Geth's underlying LevelDB datastore and persists Ethereum blocks, 
-transactions, receipts and logs into VulcanizeDB tables named `blocks`, `uncles`, 
-`full_sync_transactions`, `full_sync_receipts` and `logs` respectively.
-
-#### Usage
-1. Ensure the Ethereum node you're point at is not running, and that it has synced to the desired block height.
-1. Run `./vulcanizedb coldImport --config <config.toml>`
-1. Optional flags:
-    - `--starting-block-number <block number>`/`-s <block number>`: block number to start syncing from
-    - `--ending-block-number <block number>`/`-e <block number>`: block number to sync to
-    - `--all`/`-a`: sync all missing blocks
-
-The config file can be formatted as follows, and must contain the LevelDB path.
-
-```toml
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    user     = "vulcanize"
-    password = "vulcanize"
-    port     = 5432
-
-[client]
-    leveldbpath = "/Users/user/Library/Ethereum/geth/chaindata"
-```

documentation/generic-transformer.md

@@ -1,160 +0,0 @@
-# Generic Transformer
-The `contractWatcher` command is a built-in generic contract watcher. It can watch any and all events for a given contract provided the contract's ABI is available.
-It also provides some state variable coverage by automating polling of public methods, with some restrictions:
-1. The method must have 2 or less arguments
-1. The method's arguments must all be of type address or bytes32 (hash)
-1. The method must return a single value
-
-This command operates in two modes- `header` and `full`- which require a header or full-synced vulcanizeDB, respectively.
-
-This command requires the contract ABI be available on Etherscan if it is not provided in the config file by the user.
-
-If method polling is turned on we require an archival node at the ETH ipc endpoint in our config, whether or not we are operating in `header` or `full` mode.
-Otherwise we only need to connect to a full node.
-
-## Configuration
-This command takes a config of the form:
-
-```toml
-  [database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    port     = 5432
-
-  [client]
-    ipcPath  = "/Users/user/Library/Ethereum/geth.ipc"
-
-  [contract]
-    network  = ""
-    addresses  = [
-        "contractAddress1",
-        "contractAddress2"
-    ]
-    [contract.contractAddress1]
-        abi    = 'ABI for contract 1'
-        startingBlock = 982463
-    [contract.contractAddress2]
-        abi    = 'ABI for contract 2'
-        events = [
-            "event1",
-            "event2"
-        ]
-		eventArgs = [
-			"arg1",
-			"arg2"
-		]
-        methods = [
-            "method1",
-			"method2"
-        ]
-		methodArgs = [
-			"arg1",
-			"arg2"
-		]
-        startingBlock = 4448566
-        piping = true
-````
-
-- The `contract` section defines which contracts we want to watch and with which conditions.
-- `network` is only necessary if the ABIs are not provided and wish to be fetched from Etherscan.
-    - Empty or nil string indicates mainnet
-    - "ropsten", "kovan", and "rinkeby" indicate their respective networks
-- `addresses` lists the contract addresses we are watching and is used to load their individual configuration parameters
-- `contract.<contractAddress>` are the sub-mappings which contain the parameters specific to each contract address
-    - `abi` is the ABI for the contract; if none is provided the application will attempt to fetch one from Etherscan using the provided address and network
-    - `events` is the list of events to watch
-        - If this field is omitted or no events are provided then by defualt ALL events extracted from the ABI will be watched
-        - If event names are provided then only those events will be watched
-    - `eventArgs` is the list of arguments to filter events with
-        - If this field is omitted or no eventArgs are provided then by default watched events are not filtered by their argument values
-        - If eventArgs are provided then only those events which emit at least one of these values as an argument are watched
-    - `methods` is the list of methods to poll
-        - If this is omitted or no methods are provided then by default NO methods are polled
-        - If method names are provided then those methods will be polled, provided
-            1) Method has two or less arguments
-            1) Arguments are all of address or hash types
-            1) Method returns a single value
-    - `methodArgs` is the list of arguments to limit polling methods to
-        - If this field is omitted or no methodArgs are provided then by default methods will be polled with every combination of the appropriately typed values that have been collected from watched events
-        - If methodArgs are provided then only those values will be used to poll methods
-    - `startingBlock` is the block we want to begin watching the contract, usually the deployment block of that contract
-    - `piping` is a boolean flag which indicates whether or not we want to pipe return method values forward as arguments to subsequent method calls
-
-At the very minimum, for each contract address an ABI and a starting block number need to be provided (or just the starting block if the ABI can be reliably fetched from Etherscan).
-With just this information we will be able to watch all events at the contract, but with no additional filters and no method polling.
-
-## Output
-
-Transformed events and polled method results are committed to Postgres in schemas and tables generated according to the contract abi.      
-
-Schemas are created for each contract using the naming convention `<sync-type>_<lowercase contract-address>`
-Under this schema, tables are generated for watched events as `<lowercase event name>_event` and for polled methods as `<lowercase method name>_method`  
-The 'method' and 'event' identifiers are tacked onto the end of the table names to prevent collisions between methods and events of the same lowercase name    
-
-## Example:
-
-Modify `./environments/example.toml` to replace the empty `ipcPath` with a path that points to an ethjson_rpc endpoint (e.g. a local geth node ipc path or an Infura url).
-This endpoint should be for an archival eth node if we want to perform method polling as this configuration is currently set up to do. To work with a non-archival full node,
-remove the `balanceOf` method from the `0x8dd5fbce2f6a956c3022ba3663759011dd51e73e` (TrueUSD) contract.
-
-If you are operating a header sync vDB, run:
-
- `./vulcanizedb contractWatcher --config=./environments/example.toml --mode=header`
-
-If instead you are operating a full sync vDB and provided an archival node IPC path, run in full mode:
-
- `./vulcanizedb contractWatcher --config=./environments/example.toml --mode=full`
-
-This will run the contractWatcher and configures it to watch the contracts specified in the config file. Note that
-by default we operate in `header` mode but the flag is included here to demonstrate its use.
-
-The example config we link to in this example watches two contracts, the ENS Registry (0x314159265dD8dbb310642f98f50C066173C1259b) and TrueUSD (0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E).
-
-Because the ENS Registry is configured with only an ABI and a starting block, we will watch all events for this contract and poll none of its methods. Note that the ENS Registry is an example
-of a contract which does not have its ABI available over Etherscan and must have it included in the config file.
-
-The TrueUSD contract is configured with two events (`Transfer` and `Mint`) and a single method (`balanceOf`), as such it will watch these two events and use any addresses it collects emitted from them
-to poll the `balanceOf` method with those addresses at every block. Note that we do not provide an ABI for TrueUSD as its ABI can be fetched from Etherscan.
-
-For the ENS contract, it produces and populates a schema with four tables"
-`header_0x314159265dd8dbb310642f98f50c066173c1259b.newowner_event`
-`header_0x314159265dd8dbb310642f98f50c066173c1259b.newresolver_event`
-`header_0x314159265dd8dbb310642f98f50c066173c1259b.newttl_event`
-`header_0x314159265dd8dbb310642f98f50c066173c1259b.transfer_event`
-
-For the TrusUSD contract, it produces and populates a schema with three tables:
-
-`header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event`
-`header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.mint_event`
-`header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method`
-
-Column ids and types for these tables are generated based on the event and method argument names and types and method return types, resulting in tables such as:
-
-Table "header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event"
-
-|  Column    |         Type          | Collation | Nullable |                                           Default                                            | Storage  | Stats target | Description  
-|:----------:|:---------------------:|:---------:|:--------:|:--------------------------------------------------------------------------------------------:|:--------:|:------------:|:-----------:|
-| id         | integer               |           | not null | nextval('header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event_id_seq'::regclass) | plain    |              |             |
-| header_id  | integer               |           | not null |                                                                                              | plain    |              |             |
-| token_name | character varying(66) |           | not null |                                                                                              | extended |              |             |
-| raw_log    | jsonb                 |           |          |                                                                                              | extended |              |             |
-| log_idx    | integer               |           | not null |                                                                                              | plain    |              |             |
-| tx_idx     | integer               |           | not null |                                                                                              | plain    |              |             |
-| from_      | character varying(66) |           | not null |                                                                                              | extended |              |             |
-| to_        | character varying(66) |           | not null |                                                                                              | extended |              |             |
-| value_     | numeric               |           | not null |                                                                                              | main     |              |             |
-
-
-Table "header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method"
-
-|   Column   |         Type          | Collation | Nullable |                                            Default                                             | Storage  | Stats target | Description |
-|:----------:|:---------------------:|:---------:|:--------:|:----------------------------------------------------------------------------------------------:|:--------:|:------------:|:-----------:|
-| id         | integer               |           | not null | nextval('header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method_id_seq'::regclass) | plain    |              |             |
-| token_name | character varying(66) |           | not null |                                                                                                | extended |              |             |
-| block      | integer               |           | not null |                                                                                                | plain    |              |             |
-| who_       | character varying(66) |           | not null |                                                                                                | extended |              |             |
-| returned   | numeric               |           | not null |                                                                                                | main     |              |             |
-  
-The addition of '_' after table names is to prevent collisions with reserved Postgres words.
-
-Also notice that the contract address used for the schema name has been down-cased.

documentation/postgraphile.md

@@ -1,34 +0,0 @@
-# Postgraphile
-
-You can expose VulcanizeDB data via [Postgraphile](https://github.com/graphile/postgraphile).
-Check out [their documentation](https://www.graphile.org/postgraphile/) for the most up-to-date instructions on installing, running, and customizing Postgraphile.
-
-## Simple Setup
-
-As of April 30, 2019, you can run Postgraphile pointed at the default `vulcanize_public` database with the following commands:
-
-```
-npm install -g postgraphile
-postgraphile --connection postgres://localhost/vulcanize_public --schema=public,custom --disable-default-mutations --no-ignore-rbac
-```
-
-Arguments:
-- `--connection` specifies the database. The above command connects to the default `vulcanize_public` database 
-defined in [the example config](../environments/public.toml.example).
-- `--schema` defines what schema(s) to expose. The above exposes the `public` schema (for core VulcanizeDB data) as well as a `custom` schema (where `custom` is the name of a schema defined in executed transformers).
-- `--disable-default-mutations` prevents Postgraphile from exposing create, update, and delete operations on your data, which are otherwise enabled by default.
-- `--no-ignore-rbac` ensures that Postgraphile will only expose the tables, columns, fields, and query functions that the user has explicit access to.
-
-## Customizing Postgraphile
-
-By default, Postgraphile will expose queries for all tables defined in your chosen database/schema(s), including [filtering](https://www.graphile.org/postgraphile/filtering/) and [auto-discovered relations](https://www.graphile.org/postgraphile/relations/).
-
-If you'd like to expose more customized windows into your data, there are some techniques you can apply when writing migrations:
-
-- [Computed columns](https://www.graphile.org/postgraphile/computed-columns/) enable you to derive additional fields from types defined in your database.
-For example, you could write a function to expose a block header's state root over Postgraphile with a computed column - without modifying the `public.headers` table.
-- [Custom queries](https://www.graphile.org/postgraphile/custom-queries/) enable you to provide on-demand access to more complex data (e.g. the product of joining and filtering several tables' data based on a passed argument).
-For example, you could write a custom query to get the block timestamp for every transaction originating from a given address.
-- [Subscriptions](https://www.graphile.org/postgraphile/subscriptions/) enable you to publish data as it is coming into your database.
-
-The above list is not exhaustive - please see the Postgraphile documentation for a more comprehensive and up-to-date description of available features.

documentation/resync.md

@@ -0,0 +1,73 @@
+## ipfs-blockchain-watcher resync
+The `resync` command is made available for directing the resyncing of ipfs-blockchain-watcherdata within specified ranges.
+It also contains a utility for cleaning out old data, and resetting the validation level of data.
+
+### Rational
+
+Manual resyncing of data is useful when we want to re-validate data within specific ranges using a new source.
+
+Cleaning out data is useful when we need to remove bad/deprecated data or prepare for breaking changes to the db schemas.
+
+Resetting the validation level of data is useful for designating ranges of data for resyncing by an ongoing ipfs-blockchain-watcher
+backfill process.
+
+### Command
+
+Usage: `./ipfs-blockchain-watcher resync --config={config.toml}`
+
+Configuration can also be done through CLI options and/or environmental variables.
+CLI options can be found using `./ipfs-blockchain-watcher resync --help`.
+
+### Config
+
+Below is the set of universal config parameters for the resync command, in .toml form, with the respective environmental variables commented to the side.
+This set of parameters needs to be set no matter the chain type.
+
+```toml
+[database]
+    name     = "vulcanize_public" # $DATABASE_NAME
+    hostname = "localhost" # $DATABASE_HOSTNAME
+    port     = 5432 # $DATABASE_PORT
+    user     = "vdbm" # $DATABASE_USER
+    password = "" # $DATABASE_PASSWORD
+
+[ipfs]
+    path = "~/.ipfs" # $IPFS_PATH
+    
+[resync]
+    chain = "ethereum" # $RESYNC_CHAIN
+    type = "state" # $RESYNC_TYPE
+    start = 0 # $RESYNC_START
+    stop = 1000 # $RESYNC_STOP
+    batchSize = 10 # $RESYNC_BATCH_SIZE
+    batchNumber = 100 # $RESYNC_BATCH_NUMBER
+    timeout = 300 # $HTTP_TIMEOUT
+    clearOldCache = true # $RESYNC_CLEAR_OLD_CACHE
+    resetValidation = true # $RESYNC_RESET_VALIDATION
+```
+
+Additional parameters need to be set depending on the specific chain.
+
+For Bitcoin: 
+
+```toml
+[bitcoin]
+    httpPath = "127.0.0.1:8332" # $BTC_HTTP_PATH
+    pass = "password" # $BTC_NODE_PASSWORD
+    user = "username" # $BTC_NODE_USER
+    nodeID = "ocd0" # $BTC_NODE_ID
+    clientName = "Omnicore" # $BTC_CLIENT_NAME
+    genesisBlock = "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f" # $BTC_GENESIS_BLOCK
+    networkID = "0xD9B4BEF9" # $BTC_NETWORK_ID
+```
+
+For Ethereum:
+
+```toml
+[ethereum]
+    httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH
+    nodeID = "arch1" # $ETH_NODE_ID
+    clientName = "Geth" # $ETH_CLIENT_NAME
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK
+    networkID = "1" # $ETH_NETWORK_ID
+```

documentation/watcher.md

@@ -0,0 +1,16 @@
+These are the components of a VulcanizeDB Watcher:
+* Data Fetcher/Streamer sources:
+  * go-ethereum
+  * bitcoind
+  * btcd
+  * IPFS
+* Transformers contain:
+  * converter
+  * publisher
+  * indexer
+* Endpoints contain:
+  * api
+  * backend
+  * filterer
+  * retriever
+    * ipld_server

environments/composeAndExecuteAccountTransformer.toml

@@ -1,38 +0,0 @@
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    port     = 5432
-
-[client]
-    ipcPath = ""
-
-[exporter]
-    home     = "github.com/vulcanize/vulcanizedb"
-    name     = "accountTransformerExporter"
-    save     = false
-    transformerNames = [
-        "account"
-    ]
-    [exporter.account]
-        path = "transformers/account/light/initializer"
-        type = "eth_contract"
-        repository = "github.com/vulcanize/account_transformers"
-        migrations = "db/migrations"
-        rank = "0"
-
-[token]
-    addresses = [
-        "0x58b6A8A3302369DAEc383334672404Ee733aB239",
-        "0x862Da0A691bb0b74038377295f8fF523D0493eB4",
-    ]
-    [token.equivalents]
-        0x0000000000085d4780B73119b644AE5ecd22b376 = [
-           "0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E"
-        ]
-        0x58b6A8A3302369DAEc383334672404Ee733aB239 = [
-           "0x8e306b005773bee6bA6A6e8972Bc79D766cC15c8"
-        ]
-
-[account]
-    start = 0
-    addresses = []

environments/example.toml

@@ -1,26 +0,0 @@
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    port     = 5432
-
-[client]
-    ipcPath  = ""
-
-[contract]
-    network  = ""
-    addresses  = [
-        "0x314159265dD8dbb310642f98f50C066173C1259b",
-        "0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E"
-    ]
-    [contract.0x314159265dD8dbb310642f98f50C066173C1259b]
-        abi    = '[{"constant":true,"inputs":[{"name":"node","type":"bytes32"}],"name":"resolver","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"node","type":"bytes32"}],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"label","type":"bytes32"},{"name":"owner","type":"address"}],"name":"setSubnodeOwner","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"ttl","type":"uint64"}],"name":"setTTL","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"node","type":"bytes32"}],"name":"ttl","outputs":[{"name":"","type":"uint64"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"resolver","type":"address"}],"name":"setResolver","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"owner","type":"address"}],"name":"setOwner","outputs":[],"payable":false,"type":"function"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"owner","type":"address"}],"name":"Transfer","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":true,"name":"label","type":"bytes32"},{"indexed":false,"name":"owner","type":"address"}],"name":"NewOwner","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"resolver","type":"address"}],"name":"NewResolver","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"ttl","type":"uint64"}],"name":"NewTTL","type":"event"}]'
-        startingBlock = 3327417
-    [contract.0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E]
-        events = [
-            "Transfer",
-            "Issue"
-        ]
-        methods = [
-            "balanceOf"
-        ]
-        startingBlock = 5197514

environments/infura.toml

@@ -1,7 +0,0 @@
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    port     = 5432
-
-[client]
-    ipcPath  = ""

environments/private.toml

@@ -1,7 +0,0 @@
-[database]
-    name     = "vulcanize_private"
-    hostname = "localhost"
-    port     = 5432
-
-[client]
-    ipcPath  = "http://127.0.0.1:7545"

environments/public.toml.example

@@ -1,8 +0,0 @@
-[database]
-    name = "vulcanize_public"
-    hostname = "localhost"
-    port = 5432
-
-[client]
-    ipcPath = <local node's IPC filepath>
-    levelDbPath = <local node's LevelDB chaindata filepath>

environments/superNodeBTC.toml

@@ -0,0 +1,45 @@
+[database]
+    name     = "vulcanize_public" # $DATABASE_NAME
+    hostname = "localhost" # $DATABASE_HOSTNAME
+    port     = 5432 # $DATABASE_PORT
+    user     = "vdbm" # $DATABASE_USER
+    password = "" # $DATABASE_PASSWORD
+
+    [database.sync]
+        maxIdle = 1
+    [database.backFill]
+        maxIdle = 5
+
+[resync]
+    chain = "bitcoin" # $RESYNC_CHAIN
+    type = "full" # $RESYNC_TYPE
+    start = 0 # $RESYNC_START
+    stop = 0 # $RESYNC_STOP
+    batchSize = 5 # $RESYNC_BATCH_SIZE
+    batchNumber = 5 # $RESYNC_BATCH_NUMBER
+    clearOldCache = false # $RESYNC_CLEAR_OLD_CACHE
+    resetValidation = true # $RESYNC_RESET_VALIDATION
+
+[superNode]
+    chain = "bitcoin" # $SUPERNODE_CHAIN
+    server = true # $SUPERNODE_SERVER
+    ipcPath = "~/.vulcanize/vulcanize.ipc" # $SUPERNODE_IPC_PATH
+    wsPath = "127.0.0.1:8082" # $SUPERNODE_WS_PATH
+    httpPath = "127.0.0.1:8083" # $SUPERNODE_HTTP_PATH
+    sync = true # $SUPERNODE_SYNC
+    workers = 1 # $SUPERNODE_WORKERS
+    backFill = true # $SUPERNODE_BACKFILL
+    frequency = 45 # $SUPERNODE_FREQUENCY
+    batchSize = 5 # $SUPERNODE_BATCH_SIZE
+    batchNumber = 5 # $SUPERNODE_BATCH_NUMBER
+    validationLevel = 1 # $SUPERNODE_VALIDATION_LEVEL
+
+[bitcoin]
+    wsPath  = "127.0.0.1:8332" # $BTC_WS_PATH
+    httpPath = "127.0.0.1:8332" # $BTC_HTTP_PATH
+    pass = "password" # $BTC_NODE_PASSWORD
+    user = "username" # $BTC_NODE_USER
+    nodeID = "ocd0" # $BTC_NODE_ID
+    clientName = "Omnicore" # $BTC_CLIENT_NAME
+    genesisBlock = "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f" # $BTC_GENESIS_BLOCK
+    networkID = "0xD9B4BEF9" # $BTC_NETWORK_ID

environments/superNodeETH.toml

@@ -0,0 +1,45 @@
+[database]
+    name     = "vulcanize_public" # $DATABASE_NAME
+    hostname = "localhost" # $DATABASE_HOSTNAME
+    port     = 5432 # $DATABASE_PORT
+    user     = "vdbm" # $DATABASE_USER
+    password = "" # $DATABASE_PASSWORD
+
+    [database.sync]
+        maxIdle = 1
+    [database.backFill]
+        maxIdle = 5
+
+[resync]
+    chain = "ethereum" # $RESYNC_CHAIN
+    type = "state" # $RESYNC_TYPE
+    start = 0 # $RESYNC_START
+    stop = 0 # $RESYNC_STOP
+    batchSize = 5 # $RESYNC_BATCH_SIZE
+    batchNumber = 5 # $RESYNC_BATCH_NUMBER
+    timeout = 300 # $HTTP_TIMEOUT
+    clearOldCache = true # $RESYNC_CLEAR_OLD_CACHE
+    resetValidation = true # $RESYNC_RESET_VALIDATION
+
+[superNode]
+    chain = "ethereum" # $SUPERNODE_CHAIN
+    server = true # $SUPERNODE_SERVER
+    ipcPath = "~/.vulcanize/vulcanize.ipc" # $SUPERNODE_IPC_PATH
+    wsPath = "127.0.0.1:8081" # $SUPERNODE_WS_PATH
+    httpPath = "127.0.0.1:8082" # $SUPERNODE_HTTP_PATH
+    sync = true # $SUPERNODE_SYNC
+    workers = 1 # $SUPERNODE_WORKERS
+    backFill = true # $SUPERNODE_BACKFILL
+    frequency = 15 # $SUPERNODE_FREQUENCY
+    batchSize = 5 # $SUPERNODE_BATCH_SIZE
+    batchNumber = 5 # $SUPERNODE_BATCH_NUMBER
+    timeout = 300 # $HTTP_TIMEOUT
+    validationLevel = 1 # $SUPERNODE_VALIDATION_LEVEL
+
+[ethereum]
+    wsPath  = "127.0.0.1:8546" # $ETH_WS_PATH
+    httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH
+    nodeID = "arch1" # $ETH_NODE_ID
+    clientName = "Geth" # $ETH_CLIENT_NAME
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK
+    networkID = "1" # $ETH_NETWORK_ID

environments/superNodeSubscription.toml

@@ -0,0 +1,30 @@
+[superNode]
+    [superNode.ethSubscription]
+        historicalData = false
+        historicalDataOnly = false
+        startingBlock = 0
+        endingBlock = 0
+        wsPath = "ws://127.0.0.1:8080"
+        [superNode.ethSubscription.headerFilter]
+            off = false
+            uncles = false
+        [superNode.ethSubscription.txFilter]
+            off = false
+            src = []
+            dst = []
+        [superNode.ethSubscription.receiptFilter]
+            off = false
+            contracts = []
+            topic0s = []
+            topic1s = []
+            topic2s = []
+            topic3s = []
+        [superNode.ethSubscription.stateFilter]
+            off = false
+            addresses = []
+           intermediateNodes = false
+        [superNode.ethSubscription.storageFilter]
+            off = true
+            addresses = []
+            storageKeys = []
+            intermediateNodes = false