Commandline (#135)

* Add cmd line tool and Makefile

* Add shared utils pkg

* Add cmdline README

* Update godo for new structure

.gitignore

@@ -8,3 +8,4 @@ Vagrantfile
 vagrant_bootstrap.sh
 .vagrant
 test_scripts/
+vulcanize

Gododir/main.go

@@ -5,8 +5,8 @@ import (
 
 	"fmt"
 
-	"github.com/8thlight/vulcanizedb/cmd"
 	"github.com/8thlight/vulcanizedb/pkg/config"
+	"github.com/8thlight/vulcanizedb/utils"
 	do "gopkg.in/godo.v2"
 )
 
@@ -79,7 +79,7 @@ func tasks(p *do.Project) {
 
 	p.Task("migrate", nil, func(context *do.Context) {
 		environment := parseEnvironment(context)
-		cfg := cmd.LoadConfig(environment)
+		cfg := utils.LoadConfig(environment)
 		connectString := config.DbConnectionString(cfg.Database)
 		migrate := fmt.Sprintf("migrate -database '%s' -path ./db/migrations up", connectString)
 		dumpSchema := fmt.Sprintf("pg_dump -O -s %s > db/schema.sql", cfg.Database.Name)
@@ -89,7 +89,7 @@ func tasks(p *do.Project) {
 
 	p.Task("rollback", nil, func(context *do.Context) {
 		environment := parseEnvironment(context)
-		cfg := cmd.LoadConfig(environment)
+		cfg := utils.LoadConfig(environment)
 		connectString := config.DbConnectionString(cfg.Database)
 		migrate := fmt.Sprintf("migrate -database '%s' -path ./db/migrations down 1", connectString)
 		dumpSchema := fmt.Sprintf("pg_dump -O -s %s > db/schema.sql", cfg.Database.Name)

Gopkg.lock

@@ -31,6 +31,12 @@
   revision = "4bb3c89d44e372e6a9ab85a8be0c9345265c763a"
   version = "v1.7.3"
 
+[[projects]]
+  name = "github.com/fsnotify/fsnotify"
+  packages = ["."]
+  revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
+  version = "v1.4.7"
+
 [[projects]]
   name = "github.com/go-stack/stack"
   packages = ["."]
@@ -49,6 +55,12 @@
   packages = ["."]
   revision = "553a641470496b2327abcac10b36396bd98e45c9"
 
+[[projects]]
+  branch = "master"
+  name = "github.com/hashicorp/hcl"
+  packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"]
+  revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8"
+
 [[projects]]
   branch = "master"
   name = "github.com/howeyc/gopass"
@@ -61,6 +73,12 @@
   packages = [".","dcps/internetgateway1","dcps/internetgateway2","httpu","scpd","soap","ssdp"]
   revision = "dceda08e705b2acee36aab47d765ed801f64cfc7"
 
+[[projects]]
+  name = "github.com/inconshreveable/mousetrap"
+  packages = ["."]
+  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
+  version = "v1.0"
+
 [[projects]]
   name = "github.com/jackpal/go-nat-pmp"
   packages = ["."]
@@ -79,6 +97,12 @@
   packages = [".","oid"]
   revision = "83612a56d3dd153a94a629cd64925371c9adad78"
 
+[[projects]]
+  name = "github.com/magiconair/properties"
+  packages = ["."]
+  revision = "d419a98cdbed11a922bf76f257b7c4be79b50e73"
+  version = "v1.7.4"
+
 [[projects]]
   name = "github.com/mattn/go-colorable"
   packages = ["."]
@@ -115,6 +139,18 @@
   revision = "00c06406c2dd2e011f153a6502a21473676db33f"
   version = "v1.0.0"
 
+[[projects]]
+  branch = "master"
+  name = "github.com/mitchellh/go-homedir"
+  packages = ["."]
+  revision = "b8bc1bf767474819792c23f32d8286a45736f1c6"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/mitchellh/mapstructure"
+  packages = ["."]
+  revision = "b4575eea38cca1123ec2dc90c26529b5c5acfcff"
+
 [[projects]]
   name = "github.com/nozzle/throttler"
   packages = ["."]
@@ -133,6 +169,12 @@
   revision = "c893efa28eb45626cdaa76c9f653b62488858837"
   version = "v1.2.0"
 
+[[projects]]
+  name = "github.com/pelletier/go-toml"
+  packages = ["."]
+  revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8"
+  version = "v1.1.0"
+
 [[projects]]
   branch = "master"
   name = "github.com/rcrowley/go-metrics"
@@ -145,6 +187,42 @@
   revision = "7af7a1e09ba336d2ea14b1ce73bf693c6837dbf6"
   version = "v1.2"
 
+[[projects]]
+  name = "github.com/spf13/afero"
+  packages = [".","mem"]
+  revision = "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c"
+  version = "v1.0.2"
+
+[[projects]]
+  name = "github.com/spf13/cast"
+  packages = ["."]
+  revision = "acbeb36b902d72a7a4c18e8f3241075e7ab763e4"
+  version = "v1.1.0"
+
+[[projects]]
+  name = "github.com/spf13/cobra"
+  packages = ["."]
+  revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b"
+  version = "v0.0.1"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/spf13/jwalterweatherman"
+  packages = ["."]
+  revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394"
+
+[[projects]]
+  name = "github.com/spf13/pflag"
+  packages = ["."]
+  revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66"
+  version = "v1.0.0"
+
+[[projects]]
+  name = "github.com/spf13/viper"
+  packages = ["."]
+  revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7"
+  version = "v1.0.0"
+
 [[projects]]
   branch = "master"
   name = "github.com/syndtr/goleveldb"
@@ -172,7 +250,7 @@
 [[projects]]
   branch = "master"
   name = "golang.org/x/text"
-  packages = ["encoding","encoding/charmap","encoding/htmlindex","encoding/internal","encoding/internal/identifier","encoding/japanese","encoding/korean","encoding/simplifiedchinese","encoding/traditionalchinese","encoding/unicode","internal/gen","internal/tag","internal/utf8internal","language","runes","transform","unicode/cldr"]
+  packages = ["encoding","encoding/charmap","encoding/htmlindex","encoding/internal","encoding/internal/identifier","encoding/japanese","encoding/korean","encoding/simplifiedchinese","encoding/traditionalchinese","encoding/unicode","internal/gen","internal/tag","internal/triegen","internal/ucd","internal/utf8internal","language","runes","transform","unicode/cldr","unicode/norm"]
   revision = "be25de41fadfae372d6470bda81ca6beb55ef551"
 
 [[projects]]
@@ -208,6 +286,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "2d7b9c5c88a94f3384b0cd754d35a3d7822a5858f439aaafe8c6477fb7c24f63"
+  inputs-digest = "46080c5bb453cb31ea731741d5defda225e2dc75bf7c11179d69422c82de37ca"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -32,3 +32,7 @@
 [[constraint]]
   branch = "master"
   name = "github.com/lib/pq"
+
+[[constraint]]
+  name = "github.com/spf13/cobra"
+  version = "0.0.1"

Makefile

@@ -0,0 +1,36 @@
+HOST_NAME ?= $(DB_HOST_NAME) 
+PORT ?= $(DB_PORT)  
+NAME ?= $(DB_NAME) 
+CONNECT_STRING=postgresql://$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable
+
+$(MATTESMIGRATE):
+	go get -u -d github.com/mattes/migrate/cli github.com/lib/pq
+	go build -tags 'postgres' -o /usr/local/bin/migrate github.com/mattes/migrate/cli
+
+$(DEP):
+	go get -u github.com/golang/dep/cmd/dep
+
+$(GINKGO):
+	go get -u github.com/onsi/ginkgo/ginkgo  
+
+checkdbvars:
+	test -n "$(HOST_NAME)" # $$HOST_NAME
+	test -n "$(PORT)" # $$PORT
+	test -n "$(NAME)" # $$NAME
+
+rollback: checkdbvars
+	migrate -database $(CONNECT_STRING) -path ./db/migrations down 1
+	pg_dump -O -s $(CONNECT_STRING) > db/schema.sql
+
+migrate: $(MATTESMIGRATE) checkdbvars
+	migrate -database $(CONNECT_STRING) -path ./db/migrations up
+	pg_dump -O -s $(CONNECT_STRING) > db/schema.sql
+
+dep: $(DEP)
+	dep ensure
+
+build: dep 
+	go build -o vulcanize ./commands 
+
+test: $(GINKGO)
+	ginkgo -r

cmd/add_filter/main.go

@@ -6,9 +6,9 @@ import (
 	"io/ioutil"
 	"log"
 
-	"github.com/8thlight/vulcanizedb/cmd"
 	"github.com/8thlight/vulcanizedb/pkg/filters"
 	"github.com/8thlight/vulcanizedb/pkg/geth"
+	"github.com/8thlight/vulcanizedb/utils"
 )
 
 func main() {
@@ -17,10 +17,10 @@ func main() {
 
 	flag.Parse()
 	var logFilters filters.LogFilters
-	config := cmd.LoadConfig(*environment)
+	config := utils.LoadConfig(*environment)
 	blockchain := geth.NewBlockchain(config.Client.IPCPath)
-	repository := cmd.LoadPostgres(config.Database, blockchain.Node())
+	repository := utils.LoadPostgres(config.Database, blockchain.Node())
-	absFilePath := cmd.AbsFilePath(*filterFilePath)
+	absFilePath := utils.AbsFilePath(*filterFilePath)
 	logFilterBytes, err := ioutil.ReadFile(absFilePath)
 	if err != nil {
 		log.Fatal(err)

cmd/populate_blocks/main.go

@@ -5,18 +5,18 @@ import (
 
 	"fmt"
 
-	"github.com/8thlight/vulcanizedb/cmd"
 	"github.com/8thlight/vulcanizedb/pkg/geth"
 	"github.com/8thlight/vulcanizedb/pkg/history"
+	"github.com/8thlight/vulcanizedb/utils"
 )
 
 func main() {
 	environment := flag.String("environment", "", "Environment name")
 	startingBlockNumber := flag.Int("starting-number", -1, "First block to fill from")
 	flag.Parse()
-	config := cmd.LoadConfig(*environment)
+	config := utils.LoadConfig(*environment)
 	blockchain := geth.NewBlockchain(config.Client.IPCPath)
-	repository := cmd.LoadPostgres(config.Database, blockchain.Node())
+	repository := utils.LoadPostgres(config.Database, blockchain.Node())
 	numberOfBlocksCreated := history.PopulateMissingBlocks(blockchain, repository, int64(*startingBlockNumber))
 	fmt.Printf("Populated %d blocks", numberOfBlocksCreated)
 }

cmd/run/main.go

@@ -7,9 +7,9 @@ import (
 
 	"os"
 
-	"github.com/8thlight/vulcanizedb/cmd"
 	"github.com/8thlight/vulcanizedb/pkg/geth"
 	"github.com/8thlight/vulcanizedb/pkg/history"
+	"github.com/8thlight/vulcanizedb/utils"
 )
 
 const (
@@ -23,9 +23,9 @@ func main() {
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()
 
-	config := cmd.LoadConfig(*environment)
+	config := utils.LoadConfig(*environment)
 	blockchain := geth.NewBlockchain(config.Client.IPCPath)
-	repository := cmd.LoadPostgres(config.Database, blockchain.Node())
+	repository := utils.LoadPostgres(config.Database, blockchain.Node())
 	validator := history.NewBlockValidator(blockchain, repository, 15)
 
 	for range ticker.C {

cmd/show_contract_summary/main.go

@@ -9,9 +9,9 @@ import (
 
 	"strings"
 
-	"github.com/8thlight/vulcanizedb/cmd"
 	"github.com/8thlight/vulcanizedb/pkg/contract_summary"
 	"github.com/8thlight/vulcanizedb/pkg/geth"
+	"github.com/8thlight/vulcanizedb/utils"
 )
 
 func main() {
@@ -21,10 +21,10 @@ func main() {
 	flag.Parse()
 
 	contractHashLowered := strings.ToLower(*contractHash)
-	config := cmd.LoadConfig(*environment)
+	config := utils.LoadConfig(*environment)
 	blockchain := geth.NewBlockchain(config.Client.IPCPath)
-	repository := cmd.LoadPostgres(config.Database, blockchain.Node())
+	repository := utils.LoadPostgres(config.Database, blockchain.Node())
-	blockNumber := cmd.RequestedBlockNumber(_blockNumber)
+	blockNumber := utils.RequestedBlockNumber(_blockNumber)
 
 	contractSummary, err := contract_summary.NewSummary(blockchain, repository, contractHashLowered, blockNumber)
 	if err != nil {

cmd/vulcanize_db/main.go

@@ -7,11 +7,11 @@ import (
 
 	"os"
 
-	"github.com/8thlight/vulcanizedb/cmd"
 	"github.com/8thlight/vulcanizedb/pkg/core"
 	"github.com/8thlight/vulcanizedb/pkg/geth"
 	"github.com/8thlight/vulcanizedb/pkg/history"
 	"github.com/8thlight/vulcanizedb/pkg/repositories"
+	"github.com/8thlight/vulcanizedb/utils"
 )
 
 const (
@@ -32,9 +32,9 @@ func main() {
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()
 
-	config := cmd.LoadConfig(*environment)
+	config := utils.LoadConfig(*environment)
 	blockchain := geth.NewBlockchain(config.Client.IPCPath)
-	repository := cmd.LoadPostgres(config.Database, blockchain.Node())
+	repository := utils.LoadPostgres(config.Database, blockchain.Node())
 	validator := history.NewBlockValidator(blockchain, repository, 15)
 
 	missingBlocksPopulated := make(chan int)

cmd/watch_contract/main.go

@@ -5,9 +5,9 @@ import (
 
 	"strings"
 
-	"github.com/8thlight/vulcanizedb/cmd"
 	"github.com/8thlight/vulcanizedb/pkg/core"
 	"github.com/8thlight/vulcanizedb/pkg/geth"
+	"github.com/8thlight/vulcanizedb/utils"
 )
 
 func main() {
@@ -19,10 +19,10 @@ func main() {
 	flag.Parse()
 	contractHashLowered := strings.ToLower(*contractHash)
 
-	contractAbiString := cmd.GetAbi(*abiFilepath, contractHashLowered, *network)
+	contractAbiString := utils.GetAbi(*abiFilepath, contractHashLowered, *network)
-	config := cmd.LoadConfig(*environment)
+	config := utils.LoadConfig(*environment)
 	blockchain := geth.NewBlockchain(config.Client.IPCPath)
-	repository := cmd.LoadPostgres(config.Database, blockchain.Node())
+	repository := utils.LoadPostgres(config.Database, blockchain.Node())
 	watchedContract := core.Contract{
 		Abi:  contractAbiString,
 		Hash: contractHashLowered,

commands/README.md

@@ -0,0 +1,90 @@
+# Vulcanize DB
+
+[![Build Status](https://travis-ci.com/8thlight/vulcanizedb.svg?token=GKv2Y33qsFnfYgejjvYx&branch=master)](https://travis-ci.com/8thlight/vulcanizedb)
+
+## Development Setup
+
+### Dependencies
+
+ - Go 1.9+
+ - Postgres 10 
+ - Go Ethereum 
+    - https://ethereum.github.io/go-ethereum/downloads/ 
+    
+### Cloning the Repository
+
+1. `git config --global url."git@github.com:".insteadOf "https://github.com/"`
+    - By default, `go get` does not work for private GitHub repos. This will fix that.
+2. `go get github.com/8thlight/vulcanizedb`
+3. `cd $GOPATH/src/github.com/8thlight/vulcanizedb`
+4. `dep ensure`
+
+### Setting up the Databases
+
+1. Install Postgres
+2. Create a superuser for yourself and make sure `psql --list` works without prompting for a password.
+3. `createdb vulcanize_private`
+4. `cd $GOPATH/src/github.com/8thlight/vulcanizedb`
+5. `HOST_NAME=localhost NAME=vulcanize_public PORT=5432 make migrate`
+    * See below for configuring additional environments
+
+Adding a new migration: `./scripts/create_migration <migration-name>`
+
+### Building 
+1. `make build`
+
+### Creating/Using a Private Blockchain
+
+Syncing the public blockchain takes many hours for the initial sync and will download 20+ GB of data.
+Here are some instructions for creating a private blockchain that does not depend on having a network connection.
+
+1. Run `./scripts/setup` to create a private blockchain with a new account.
+    * This will result in a warning.
+2. Run `./scripts/start_private_blockchain`.
+3. Run `godo run -- --environment=private` to start listener.
+
+### Connecting to the Public Blockchain
+
+`./scripts/start_blockchain`
+
+### IPC File Paths
+
+The default location for Ethereum is:
+ - `$HOME/Library/Ethereum` for Mac
+ - `$HOME/.ethereum` for Ubuntu
+ - `$GOPATH/src/gihub.com/8thlight/vulcanizedb/test_data_dir/geth.ipc` for private blockchain.
+
+**Note the location of the ipc file is outputted when you connect to geth. It is needed to for configuration**
+
+## Start Vulcanize DB
+1. Start geth
+2. In a separate terminal start vulcanize_db
+    - `vulcanize sync --config <config.toml> --starting-block-number <block-numbe>`
+
+## Watch contract events
+1. Start geth
+2. In a separate terminal start vulcanize_db
+    - `vulcanize sync --config <config.toml> --starting-block-number <block-numbe>`
+3. Create event filter 
+    - `vulcanize addFilter --config <config.toml> --filter-filepath <filter.json>`
+     
+### Configuring Additional Environments
+
+You can create configuration files for additional environments.
+
+ * Among other things, it will require the IPC file path
+ * See `environments/private.toml` for an example
+ * You will need to do this if you want to run a node connecting to the public blockchain
+
+## Running the Tests
+
+### Unit Tests
+
+1. `go test ./pkg/...`
+
+### Integration Test
+
+In order to run the integration tests, you will need to run them against a real blockchain. At the moment the integration tests require [Geth v1.7.2](https://ethereum.github.io/go-ethereum/downloads/) as they depend on the `--dev` mode, which changed in v1.7.3 
+
+1. Run `./scripts/start_private_blockchain` as a separate process.
+2. `go test ./...` to run all tests.

commands/cmd/addFilter.go

@@ -0,0 +1,75 @@
+package cmd
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"log"
+
+	"github.com/8thlight/vulcanizedb/pkg/filters"
+	"github.com/8thlight/vulcanizedb/pkg/geth"
+	"github.com/8thlight/vulcanizedb/utils"
+	"github.com/spf13/cobra"
+)
+
+// addFilterCmd represents the addFilter command
+var addFilterCmd = &cobra.Command{
+	Use:   "addFilter",
+	Short: "Adds event filter to vulcanize_db",
+	Long: `An event filter is added to the vulcanize_db. 
+All events matching the filter conitions will be tracked 
+in vulcanize. 
+
+vulcanize addFilter --config config.toml --filter-filepath filter.json
+
+The event filters are expected to match
+the format described in the ethereum RPC wiki:
+
+https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_newfilter
+
+[{
+  "fromBlock": "0x1",
+  "toBlock": "0x2",
+  "address": "0x8888f1f195afa192cfee860698584c030f4c9db1",
+  "topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", 
+             null, 
+             "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", 
+             "0x0000000000000000000000000aff3454fce5edbc8cca8697c15331677e6ebccc"]
+}]
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		addFilter()
+	},
+}
+
+var filterFilepath string
+
+func init() {
+	rootCmd.AddCommand(addFilterCmd)
+
+	addFilterCmd.PersistentFlags().StringVar(&filterFilepath, "filter-filepath", "", "path/to/filter.json")
+	addFilterCmd.MarkFlagRequired("filter-filepath")
+}
+
+func addFilter() {
+	if filterFilepath == "" {
+		log.Fatal("filter-filepath required")
+	}
+	var logFilters filters.LogFilters
+	blockchain := geth.NewBlockchain(ipc)
+	repository := utils.LoadPostgres(databaseConfig, blockchain.Node())
+	absFilePath := utils.AbsFilePath(filterFilepath)
+	logFilterBytes, err := ioutil.ReadFile(absFilePath)
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = json.Unmarshal(logFilterBytes, &logFilters)
+	if err != nil {
+		log.Fatal(err)
+	}
+	for _, filter := range logFilters {
+		err = repository.AddFilter(filter)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+}

commands/cmd/root.go

@@ -0,0 +1,74 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/8thlight/vulcanizedb/pkg/config"
+	"github.com/mitchellh/go-homedir"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var cfgFile string
+var databaseConfig config.Database
+var ipc string
+
+var rootCmd = &cobra.Command{
+	Use:              "vulcanize",
+	PersistentPreRun: database,
+}
+
+func Execute() {
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func database(cmd *cobra.Command, args []string) {
+	ipc = viper.GetString("client.ipcpath")
+	databaseConfig = config.Database{
+		Name:     viper.GetString("database.name"),
+		Hostname: viper.GetString("database.hostname"),
+		Port:     viper.GetInt("database.port"),
+	}
+	viper.Set("database.config", databaseConfig)
+}
+
+func init() {
+	cobra.OnInitialize(initConfig)
+
+	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file: (default is $HOME/.vulcanize.yaml)")
+	rootCmd.PersistentFlags().String("database-name", "vulcanize", "database: name")
+	rootCmd.PersistentFlags().Int("database-port", 5432, "database: port")
+	rootCmd.PersistentFlags().String("database-hostname", "localhost", "database: hostname")
+	rootCmd.PersistentFlags().String("client-ipcPath", "", "geth: geth.ipc file")
+
+	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
+	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
+	viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
+	viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath"))
+
+}
+
+func initConfig() {
+	if cfgFile != "" {
+		viper.SetConfigFile(cfgFile)
+	} else {
+		home, err := homedir.Dir()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		viper.AddConfigPath(home)
+		viper.SetConfigName(".vulcanize")
+	}
+
+	viper.AutomaticEnv()
+
+	if err := viper.ReadInConfig(); err == nil {
+		fmt.Printf("Using config file: %s\n\n", viper.ConfigFileUsed())
+	}
+}

commands/cmd/sync.go

@@ -0,0 +1,77 @@
+package cmd
+
+import (
+	"os"
+
+	"time"
+
+	"github.com/8thlight/vulcanizedb/pkg/core"
+	"github.com/8thlight/vulcanizedb/pkg/geth"
+	"github.com/8thlight/vulcanizedb/pkg/history"
+	"github.com/8thlight/vulcanizedb/pkg/repositories"
+	"github.com/8thlight/vulcanizedb/utils"
+	"github.com/spf13/cobra"
+)
+
+// syncCmd represents the sync command
+var syncCmd = &cobra.Command{
+	Use:   "sync",
+	Short: "Syncs vulcanize_db with local ethereum node",
+	Long: `Syncs vulcanize_db with local ethereum node. 
+vulcanize sync --startingBlockNumber 0 --config public.toml
+
+Expects ethereum node to be running and requires a .toml config:
+
+  [database]
+  name = "vulcanize_public"
+  hostname = "localhost"
+  port = 5432
+
+  [client]
+  ipcPath = "/Users/mattkrump/Library/Ethereum/geth.ipc"
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		sync()
+	},
+}
+
+const (
+	pollingInterval = 7 * time.Second
+)
+
+var startingBlockNumber int
+
+func init() {
+	rootCmd.AddCommand(syncCmd)
+
+	syncCmd.Flags().IntVarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
+}
+
+func backFillAllBlocks(blockchain core.Blockchain, repository repositories.Postgres, missingBlocksPopulated chan int, startingBlockNumber int64) {
+	go func() {
+		missingBlocksPopulated <- history.PopulateMissingBlocks(blockchain, repository, startingBlockNumber)
+	}()
+}
+
+func sync() {
+	ticker := time.NewTicker(pollingInterval)
+	defer ticker.Stop()
+
+	blockchain := geth.NewBlockchain(ipc)
+	repository := utils.LoadPostgres(databaseConfig, blockchain.Node())
+	validator := history.NewBlockValidator(blockchain, repository, 15)
+
+	missingBlocksPopulated := make(chan int)
+	_startingBlockNumber := int64(startingBlockNumber)
+	go backFillAllBlocks(blockchain, repository, missingBlocksPopulated, _startingBlockNumber)
+
+	for {
+		select {
+		case <-ticker.C:
+			window := validator.ValidateBlocks()
+			validator.Log(os.Stdout, window)
+		case <-missingBlocksPopulated:
+			go backFillAllBlocks(blockchain, repository, missingBlocksPopulated, _startingBlockNumber)
+		}
+	}
+}

commands/main.go

@@ -0,0 +1,9 @@
+package main
+
+import (
+	"github.com/8thlight/vulcanizedb/commands/cmd"
+)
+
+func main() {
+	cmd.Execute()
+}

db/migrations/1516653373_add_watched_event_logs.down.sql

@@ -1 +1,2 @@
-DROP VIEW watched_event_logs;
+DROP VIEW watched_event_logs;
+DROP VIEW block_stats;

utils/utils.go

@@ -1,4 +1,4 @@
-package cmd
+package utils
 
 import (
 	"log"
@@ -7,6 +7,8 @@ import (
 
 	"math/big"
 
+	"os"
+
 	"github.com/8thlight/vulcanizedb/pkg/config"
 	"github.com/8thlight/vulcanizedb/pkg/core"
 	"github.com/8thlight/vulcanizedb/pkg/geth"
@@ -40,7 +42,8 @@ func ReadAbiFile(abiFilepath string) string {
 
 func AbsFilePath(filePath string) string {
 	if !filepath.IsAbs(filePath) {
-		filePath = filepath.Join(config.ProjectRoot(), filePath)
+		cwd, _ := os.Getwd()
+		filePath = filepath.Join(cwd, filePath)
 	}
 	return filePath
 }