From 572023cdf53952f067874a5693882f039c4d13fe Mon Sep 17 00:00:00 2001 From: Matt K <1036969+mkrump@users.noreply.github.com> Date: Thu, 25 Jan 2018 13:21:55 -0600 Subject: [PATCH] Commandline (#135) * Add cmd line tool and Makefile * Add shared utils pkg * Add cmdline README * Update godo for new structure --- .gitignore | 1 + Gododir/main.go | 6 +- Gopkg.lock | 82 ++++++++++++++++- Gopkg.toml | 4 + Makefile | 36 ++++++++ cmd/add_filter/main.go | 8 +- cmd/populate_blocks/main.go | 6 +- cmd/run/main.go | 6 +- cmd/show_contract_summary/main.go | 8 +- cmd/vulcanize_db/main.go | 6 +- cmd/watch_contract/main.go | 8 +- commands/README.md | 90 +++++++++++++++++++ commands/cmd/addFilter.go | 75 ++++++++++++++++ commands/cmd/root.go | 74 +++++++++++++++ commands/cmd/sync.go | 77 ++++++++++++++++ commands/main.go | 9 ++ ...1516653373_add_watched_event_logs.down.sql | 3 +- {cmd => utils}/utils.go | 7 +- 18 files changed, 477 insertions(+), 29 deletions(-) create mode 100644 Makefile create mode 100644 commands/README.md create mode 100644 commands/cmd/addFilter.go create mode 100644 commands/cmd/root.go create mode 100644 commands/cmd/sync.go create mode 100644 commands/main.go rename {cmd => utils}/utils.go (95%) diff --git a/.gitignore b/.gitignore index f5b5e5ae..ffe62712 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ Vagrantfile vagrant_bootstrap.sh .vagrant test_scripts/ +vulcanize diff --git a/Gododir/main.go b/Gododir/main.go index 15200298..5f83012d 100644 --- a/Gododir/main.go +++ b/Gododir/main.go @@ -5,8 +5,8 @@ import ( "fmt" - "github.com/8thlight/vulcanizedb/cmd" "github.com/8thlight/vulcanizedb/pkg/config" + "github.com/8thlight/vulcanizedb/utils" do "gopkg.in/godo.v2" ) @@ -79,7 +79,7 @@ func tasks(p *do.Project) { p.Task("migrate", nil, func(context *do.Context) { environment := parseEnvironment(context) - cfg := cmd.LoadConfig(environment) + cfg := utils.LoadConfig(environment) connectString := config.DbConnectionString(cfg.Database) migrate := fmt.Sprintf("migrate -database '%s' -path ./db/migrations up", connectString) dumpSchema := fmt.Sprintf("pg_dump -O -s %s > db/schema.sql", cfg.Database.Name) @@ -89,7 +89,7 @@ func tasks(p *do.Project) { p.Task("rollback", nil, func(context *do.Context) { environment := parseEnvironment(context) - cfg := cmd.LoadConfig(environment) + cfg := utils.LoadConfig(environment) connectString := config.DbConnectionString(cfg.Database) migrate := fmt.Sprintf("migrate -database '%s' -path ./db/migrations down 1", connectString) dumpSchema := fmt.Sprintf("pg_dump -O -s %s > db/schema.sql", cfg.Database.Name) diff --git a/Gopkg.lock b/Gopkg.lock index 95a08ce7..a7bbf097 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -31,6 +31,12 @@ revision = "4bb3c89d44e372e6a9ab85a8be0c9345265c763a" version = "v1.7.3" +[[projects]] + name = "github.com/fsnotify/fsnotify" + packages = ["."] + revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" + version = "v1.4.7" + [[projects]] name = "github.com/go-stack/stack" packages = ["."] @@ -49,6 +55,12 @@ packages = ["."] revision = "553a641470496b2327abcac10b36396bd98e45c9" +[[projects]] + branch = "master" + name = "github.com/hashicorp/hcl" + packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"] + revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" + [[projects]] branch = "master" name = "github.com/howeyc/gopass" @@ -61,6 +73,12 @@ packages = [".","dcps/internetgateway1","dcps/internetgateway2","httpu","scpd","soap","ssdp"] revision = "dceda08e705b2acee36aab47d765ed801f64cfc7" +[[projects]] + name = "github.com/inconshreveable/mousetrap" + packages = ["."] + revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" + version = "v1.0" + [[projects]] name = "github.com/jackpal/go-nat-pmp" packages = ["."] @@ -79,6 +97,12 @@ packages = [".","oid"] revision = "83612a56d3dd153a94a629cd64925371c9adad78" +[[projects]] + name = "github.com/magiconair/properties" + packages = ["."] + revision = "d419a98cdbed11a922bf76f257b7c4be79b50e73" + version = "v1.7.4" + [[projects]] name = "github.com/mattn/go-colorable" packages = ["."] @@ -115,6 +139,18 @@ revision = "00c06406c2dd2e011f153a6502a21473676db33f" version = "v1.0.0" +[[projects]] + branch = "master" + name = "github.com/mitchellh/go-homedir" + packages = ["."] + revision = "b8bc1bf767474819792c23f32d8286a45736f1c6" + +[[projects]] + branch = "master" + name = "github.com/mitchellh/mapstructure" + packages = ["."] + revision = "b4575eea38cca1123ec2dc90c26529b5c5acfcff" + [[projects]] name = "github.com/nozzle/throttler" packages = ["."] @@ -133,6 +169,12 @@ revision = "c893efa28eb45626cdaa76c9f653b62488858837" version = "v1.2.0" +[[projects]] + name = "github.com/pelletier/go-toml" + packages = ["."] + revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8" + version = "v1.1.0" + [[projects]] branch = "master" name = "github.com/rcrowley/go-metrics" @@ -145,6 +187,42 @@ revision = "7af7a1e09ba336d2ea14b1ce73bf693c6837dbf6" version = "v1.2" +[[projects]] + name = "github.com/spf13/afero" + packages = [".","mem"] + revision = "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c" + version = "v1.0.2" + +[[projects]] + name = "github.com/spf13/cast" + packages = ["."] + revision = "acbeb36b902d72a7a4c18e8f3241075e7ab763e4" + version = "v1.1.0" + +[[projects]] + name = "github.com/spf13/cobra" + packages = ["."] + revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b" + version = "v0.0.1" + +[[projects]] + branch = "master" + name = "github.com/spf13/jwalterweatherman" + packages = ["."] + revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394" + +[[projects]] + name = "github.com/spf13/pflag" + packages = ["."] + revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66" + version = "v1.0.0" + +[[projects]] + name = "github.com/spf13/viper" + packages = ["."] + revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7" + version = "v1.0.0" + [[projects]] branch = "master" name = "github.com/syndtr/goleveldb" @@ -172,7 +250,7 @@ [[projects]] branch = "master" name = "golang.org/x/text" - packages = ["encoding","encoding/charmap","encoding/htmlindex","encoding/internal","encoding/internal/identifier","encoding/japanese","encoding/korean","encoding/simplifiedchinese","encoding/traditionalchinese","encoding/unicode","internal/gen","internal/tag","internal/utf8internal","language","runes","transform","unicode/cldr"] + packages = ["encoding","encoding/charmap","encoding/htmlindex","encoding/internal","encoding/internal/identifier","encoding/japanese","encoding/korean","encoding/simplifiedchinese","encoding/traditionalchinese","encoding/unicode","internal/gen","internal/tag","internal/triegen","internal/ucd","internal/utf8internal","language","runes","transform","unicode/cldr","unicode/norm"] revision = "be25de41fadfae372d6470bda81ca6beb55ef551" [[projects]] @@ -208,6 +286,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "2d7b9c5c88a94f3384b0cd754d35a3d7822a5858f439aaafe8c6477fb7c24f63" + inputs-digest = "46080c5bb453cb31ea731741d5defda225e2dc75bf7c11179d69422c82de37ca" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 0b5b260c..45a5aa07 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -32,3 +32,7 @@ [[constraint]] branch = "master" name = "github.com/lib/pq" + +[[constraint]] + name = "github.com/spf13/cobra" + version = "0.0.1" diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..165ee223 --- /dev/null +++ b/Makefile @@ -0,0 +1,36 @@ +HOST_NAME ?= $(DB_HOST_NAME) +PORT ?= $(DB_PORT) +NAME ?= $(DB_NAME) +CONNECT_STRING=postgresql://$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable + +$(MATTESMIGRATE): + go get -u -d github.com/mattes/migrate/cli github.com/lib/pq + go build -tags 'postgres' -o /usr/local/bin/migrate github.com/mattes/migrate/cli + +$(DEP): + go get -u github.com/golang/dep/cmd/dep + +$(GINKGO): + go get -u github.com/onsi/ginkgo/ginkgo + +checkdbvars: + test -n "$(HOST_NAME)" # $$HOST_NAME + test -n "$(PORT)" # $$PORT + test -n "$(NAME)" # $$NAME + +rollback: checkdbvars + migrate -database $(CONNECT_STRING) -path ./db/migrations down 1 + pg_dump -O -s $(CONNECT_STRING) > db/schema.sql + +migrate: $(MATTESMIGRATE) checkdbvars + migrate -database $(CONNECT_STRING) -path ./db/migrations up + pg_dump -O -s $(CONNECT_STRING) > db/schema.sql + +dep: $(DEP) + dep ensure + +build: dep + go build -o vulcanize ./commands + +test: $(GINKGO) + ginkgo -r diff --git a/cmd/add_filter/main.go b/cmd/add_filter/main.go index 026c0854..ac8f2655 100644 --- a/cmd/add_filter/main.go +++ b/cmd/add_filter/main.go @@ -6,9 +6,9 @@ import ( "io/ioutil" "log" - "github.com/8thlight/vulcanizedb/cmd" "github.com/8thlight/vulcanizedb/pkg/filters" "github.com/8thlight/vulcanizedb/pkg/geth" + "github.com/8thlight/vulcanizedb/utils" ) func main() { @@ -17,10 +17,10 @@ func main() { flag.Parse() var logFilters filters.LogFilters - config := cmd.LoadConfig(*environment) + config := utils.LoadConfig(*environment) blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) - absFilePath := cmd.AbsFilePath(*filterFilePath) + repository := utils.LoadPostgres(config.Database, blockchain.Node()) + absFilePath := utils.AbsFilePath(*filterFilePath) logFilterBytes, err := ioutil.ReadFile(absFilePath) if err != nil { log.Fatal(err) diff --git a/cmd/populate_blocks/main.go b/cmd/populate_blocks/main.go index 966b98e6..e367bf46 100644 --- a/cmd/populate_blocks/main.go +++ b/cmd/populate_blocks/main.go @@ -5,18 +5,18 @@ import ( "fmt" - "github.com/8thlight/vulcanizedb/cmd" "github.com/8thlight/vulcanizedb/pkg/geth" "github.com/8thlight/vulcanizedb/pkg/history" + "github.com/8thlight/vulcanizedb/utils" ) func main() { environment := flag.String("environment", "", "Environment name") startingBlockNumber := flag.Int("starting-number", -1, "First block to fill from") flag.Parse() - config := cmd.LoadConfig(*environment) + config := utils.LoadConfig(*environment) blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) + repository := utils.LoadPostgres(config.Database, blockchain.Node()) numberOfBlocksCreated := history.PopulateMissingBlocks(blockchain, repository, int64(*startingBlockNumber)) fmt.Printf("Populated %d blocks", numberOfBlocksCreated) } diff --git a/cmd/run/main.go b/cmd/run/main.go index a8fe9152..f95d3fe4 100644 --- a/cmd/run/main.go +++ b/cmd/run/main.go @@ -7,9 +7,9 @@ import ( "os" - "github.com/8thlight/vulcanizedb/cmd" "github.com/8thlight/vulcanizedb/pkg/geth" "github.com/8thlight/vulcanizedb/pkg/history" + "github.com/8thlight/vulcanizedb/utils" ) const ( @@ -23,9 +23,9 @@ func main() { ticker := time.NewTicker(pollingInterval) defer ticker.Stop() - config := cmd.LoadConfig(*environment) + config := utils.LoadConfig(*environment) blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) + repository := utils.LoadPostgres(config.Database, blockchain.Node()) validator := history.NewBlockValidator(blockchain, repository, 15) for range ticker.C { diff --git a/cmd/show_contract_summary/main.go b/cmd/show_contract_summary/main.go index 23ff73d1..1f03d15e 100644 --- a/cmd/show_contract_summary/main.go +++ b/cmd/show_contract_summary/main.go @@ -9,9 +9,9 @@ import ( "strings" - "github.com/8thlight/vulcanizedb/cmd" "github.com/8thlight/vulcanizedb/pkg/contract_summary" "github.com/8thlight/vulcanizedb/pkg/geth" + "github.com/8thlight/vulcanizedb/utils" ) func main() { @@ -21,10 +21,10 @@ func main() { flag.Parse() contractHashLowered := strings.ToLower(*contractHash) - config := cmd.LoadConfig(*environment) + config := utils.LoadConfig(*environment) blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) - blockNumber := cmd.RequestedBlockNumber(_blockNumber) + repository := utils.LoadPostgres(config.Database, blockchain.Node()) + blockNumber := utils.RequestedBlockNumber(_blockNumber) contractSummary, err := contract_summary.NewSummary(blockchain, repository, contractHashLowered, blockNumber) if err != nil { diff --git a/cmd/vulcanize_db/main.go b/cmd/vulcanize_db/main.go index 24cf0776..366f8615 100644 --- a/cmd/vulcanize_db/main.go +++ b/cmd/vulcanize_db/main.go @@ -7,11 +7,11 @@ import ( "os" - "github.com/8thlight/vulcanizedb/cmd" "github.com/8thlight/vulcanizedb/pkg/core" "github.com/8thlight/vulcanizedb/pkg/geth" "github.com/8thlight/vulcanizedb/pkg/history" "github.com/8thlight/vulcanizedb/pkg/repositories" + "github.com/8thlight/vulcanizedb/utils" ) const ( @@ -32,9 +32,9 @@ func main() { ticker := time.NewTicker(pollingInterval) defer ticker.Stop() - config := cmd.LoadConfig(*environment) + config := utils.LoadConfig(*environment) blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) + repository := utils.LoadPostgres(config.Database, blockchain.Node()) validator := history.NewBlockValidator(blockchain, repository, 15) missingBlocksPopulated := make(chan int) diff --git a/cmd/watch_contract/main.go b/cmd/watch_contract/main.go index 0a29f552..1a02b52d 100644 --- a/cmd/watch_contract/main.go +++ b/cmd/watch_contract/main.go @@ -5,9 +5,9 @@ import ( "strings" - "github.com/8thlight/vulcanizedb/cmd" "github.com/8thlight/vulcanizedb/pkg/core" "github.com/8thlight/vulcanizedb/pkg/geth" + "github.com/8thlight/vulcanizedb/utils" ) func main() { @@ -19,10 +19,10 @@ func main() { flag.Parse() contractHashLowered := strings.ToLower(*contractHash) - contractAbiString := cmd.GetAbi(*abiFilepath, contractHashLowered, *network) - config := cmd.LoadConfig(*environment) + contractAbiString := utils.GetAbi(*abiFilepath, contractHashLowered, *network) + config := utils.LoadConfig(*environment) blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) + repository := utils.LoadPostgres(config.Database, blockchain.Node()) watchedContract := core.Contract{ Abi: contractAbiString, Hash: contractHashLowered, diff --git a/commands/README.md b/commands/README.md new file mode 100644 index 00000000..5349be62 --- /dev/null +++ b/commands/README.md @@ -0,0 +1,90 @@ +# Vulcanize DB + +[![Build Status](https://travis-ci.com/8thlight/vulcanizedb.svg?token=GKv2Y33qsFnfYgejjvYx&branch=master)](https://travis-ci.com/8thlight/vulcanizedb) + +## Development Setup + +### Dependencies + + - Go 1.9+ + - Postgres 10 + - Go Ethereum + - https://ethereum.github.io/go-ethereum/downloads/ + +### Cloning the Repository + +1. `git config --global url."git@github.com:".insteadOf "https://github.com/"` + - By default, `go get` does not work for private GitHub repos. This will fix that. +2. `go get github.com/8thlight/vulcanizedb` +3. `cd $GOPATH/src/github.com/8thlight/vulcanizedb` +4. `dep ensure` + +### Setting up the Databases + +1. Install Postgres +2. Create a superuser for yourself and make sure `psql --list` works without prompting for a password. +3. `createdb vulcanize_private` +4. `cd $GOPATH/src/github.com/8thlight/vulcanizedb` +5. `HOST_NAME=localhost NAME=vulcanize_public PORT=5432 make migrate` + * See below for configuring additional environments + +Adding a new migration: `./scripts/create_migration ` + +### Building +1. `make build` + +### Creating/Using a Private Blockchain + +Syncing the public blockchain takes many hours for the initial sync and will download 20+ GB of data. +Here are some instructions for creating a private blockchain that does not depend on having a network connection. + +1. Run `./scripts/setup` to create a private blockchain with a new account. + * This will result in a warning. +2. Run `./scripts/start_private_blockchain`. +3. Run `godo run -- --environment=private` to start listener. + +### Connecting to the Public Blockchain + +`./scripts/start_blockchain` + +### IPC File Paths + +The default location for Ethereum is: + - `$HOME/Library/Ethereum` for Mac + - `$HOME/.ethereum` for Ubuntu + - `$GOPATH/src/gihub.com/8thlight/vulcanizedb/test_data_dir/geth.ipc` for private blockchain. + +**Note the location of the ipc file is outputted when you connect to geth. It is needed to for configuration** + +## Start Vulcanize DB +1. Start geth +2. In a separate terminal start vulcanize_db + - `vulcanize sync --config --starting-block-number ` + +## Watch contract events +1. Start geth +2. In a separate terminal start vulcanize_db + - `vulcanize sync --config --starting-block-number ` +3. Create event filter + - `vulcanize addFilter --config --filter-filepath ` + +### Configuring Additional Environments + +You can create configuration files for additional environments. + + * Among other things, it will require the IPC file path + * See `environments/private.toml` for an example + * You will need to do this if you want to run a node connecting to the public blockchain + +## Running the Tests + +### Unit Tests + +1. `go test ./pkg/...` + +### Integration Test + +In order to run the integration tests, you will need to run them against a real blockchain. At the moment the integration tests require [Geth v1.7.2](https://ethereum.github.io/go-ethereum/downloads/) as they depend on the `--dev` mode, which changed in v1.7.3 + +1. Run `./scripts/start_private_blockchain` as a separate process. +2. `go test ./...` to run all tests. diff --git a/commands/cmd/addFilter.go b/commands/cmd/addFilter.go new file mode 100644 index 00000000..21318429 --- /dev/null +++ b/commands/cmd/addFilter.go @@ -0,0 +1,75 @@ +package cmd + +import ( + "encoding/json" + "io/ioutil" + "log" + + "github.com/8thlight/vulcanizedb/pkg/filters" + "github.com/8thlight/vulcanizedb/pkg/geth" + "github.com/8thlight/vulcanizedb/utils" + "github.com/spf13/cobra" +) + +// addFilterCmd represents the addFilter command +var addFilterCmd = &cobra.Command{ + Use: "addFilter", + Short: "Adds event filter to vulcanize_db", + Long: `An event filter is added to the vulcanize_db. +All events matching the filter conitions will be tracked +in vulcanize. + +vulcanize addFilter --config config.toml --filter-filepath filter.json + +The event filters are expected to match +the format described in the ethereum RPC wiki: + +https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_newfilter + +[{ + "fromBlock": "0x1", + "toBlock": "0x2", + "address": "0x8888f1f195afa192cfee860698584c030f4c9db1", + "topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + null, + "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "0x0000000000000000000000000aff3454fce5edbc8cca8697c15331677e6ebccc"] +}] +`, + Run: func(cmd *cobra.Command, args []string) { + addFilter() + }, +} + +var filterFilepath string + +func init() { + rootCmd.AddCommand(addFilterCmd) + + addFilterCmd.PersistentFlags().StringVar(&filterFilepath, "filter-filepath", "", "path/to/filter.json") + addFilterCmd.MarkFlagRequired("filter-filepath") +} + +func addFilter() { + if filterFilepath == "" { + log.Fatal("filter-filepath required") + } + var logFilters filters.LogFilters + blockchain := geth.NewBlockchain(ipc) + repository := utils.LoadPostgres(databaseConfig, blockchain.Node()) + absFilePath := utils.AbsFilePath(filterFilepath) + logFilterBytes, err := ioutil.ReadFile(absFilePath) + if err != nil { + log.Fatal(err) + } + err = json.Unmarshal(logFilterBytes, &logFilters) + if err != nil { + log.Fatal(err) + } + for _, filter := range logFilters { + err = repository.AddFilter(filter) + if err != nil { + log.Fatal(err) + } + } +} diff --git a/commands/cmd/root.go b/commands/cmd/root.go new file mode 100644 index 00000000..69be2a30 --- /dev/null +++ b/commands/cmd/root.go @@ -0,0 +1,74 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/8thlight/vulcanizedb/pkg/config" + "github.com/mitchellh/go-homedir" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var cfgFile string +var databaseConfig config.Database +var ipc string + +var rootCmd = &cobra.Command{ + Use: "vulcanize", + PersistentPreRun: database, +} + +func Execute() { + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func database(cmd *cobra.Command, args []string) { + ipc = viper.GetString("client.ipcpath") + databaseConfig = config.Database{ + Name: viper.GetString("database.name"), + Hostname: viper.GetString("database.hostname"), + Port: viper.GetInt("database.port"), + } + viper.Set("database.config", databaseConfig) +} + +func init() { + cobra.OnInitialize(initConfig) + + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file: (default is $HOME/.vulcanize.yaml)") + rootCmd.PersistentFlags().String("database-name", "vulcanize", "database: name") + rootCmd.PersistentFlags().Int("database-port", 5432, "database: port") + rootCmd.PersistentFlags().String("database-hostname", "localhost", "database: hostname") + rootCmd.PersistentFlags().String("client-ipcPath", "", "geth: geth.ipc file") + + viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name")) + viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port")) + viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname")) + viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath")) + +} + +func initConfig() { + if cfgFile != "" { + viper.SetConfigFile(cfgFile) + } else { + home, err := homedir.Dir() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + viper.AddConfigPath(home) + viper.SetConfigName(".vulcanize") + } + + viper.AutomaticEnv() + + if err := viper.ReadInConfig(); err == nil { + fmt.Printf("Using config file: %s\n\n", viper.ConfigFileUsed()) + } +} diff --git a/commands/cmd/sync.go b/commands/cmd/sync.go new file mode 100644 index 00000000..214d0ce0 --- /dev/null +++ b/commands/cmd/sync.go @@ -0,0 +1,77 @@ +package cmd + +import ( + "os" + + "time" + + "github.com/8thlight/vulcanizedb/pkg/core" + "github.com/8thlight/vulcanizedb/pkg/geth" + "github.com/8thlight/vulcanizedb/pkg/history" + "github.com/8thlight/vulcanizedb/pkg/repositories" + "github.com/8thlight/vulcanizedb/utils" + "github.com/spf13/cobra" +) + +// syncCmd represents the sync command +var syncCmd = &cobra.Command{ + Use: "sync", + Short: "Syncs vulcanize_db with local ethereum node", + Long: `Syncs vulcanize_db with local ethereum node. +vulcanize sync --startingBlockNumber 0 --config public.toml + +Expects ethereum node to be running and requires a .toml config: + + [database] + name = "vulcanize_public" + hostname = "localhost" + port = 5432 + + [client] + ipcPath = "/Users/mattkrump/Library/Ethereum/geth.ipc" +`, + Run: func(cmd *cobra.Command, args []string) { + sync() + }, +} + +const ( + pollingInterval = 7 * time.Second +) + +var startingBlockNumber int + +func init() { + rootCmd.AddCommand(syncCmd) + + syncCmd.Flags().IntVarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from") +} + +func backFillAllBlocks(blockchain core.Blockchain, repository repositories.Postgres, missingBlocksPopulated chan int, startingBlockNumber int64) { + go func() { + missingBlocksPopulated <- history.PopulateMissingBlocks(blockchain, repository, startingBlockNumber) + }() +} + +func sync() { + ticker := time.NewTicker(pollingInterval) + defer ticker.Stop() + + blockchain := geth.NewBlockchain(ipc) + repository := utils.LoadPostgres(databaseConfig, blockchain.Node()) + validator := history.NewBlockValidator(blockchain, repository, 15) + + missingBlocksPopulated := make(chan int) + _startingBlockNumber := int64(startingBlockNumber) + go backFillAllBlocks(blockchain, repository, missingBlocksPopulated, _startingBlockNumber) + + for { + select { + case <-ticker.C: + window := validator.ValidateBlocks() + validator.Log(os.Stdout, window) + case <-missingBlocksPopulated: + go backFillAllBlocks(blockchain, repository, missingBlocksPopulated, _startingBlockNumber) + } + } +} diff --git a/commands/main.go b/commands/main.go new file mode 100644 index 00000000..50e90e3b --- /dev/null +++ b/commands/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/8thlight/vulcanizedb/commands/cmd" +) + +func main() { + cmd.Execute() +} diff --git a/db/migrations/1516653373_add_watched_event_logs.down.sql b/db/migrations/1516653373_add_watched_event_logs.down.sql index 14bb2987..294d7bfe 100644 --- a/db/migrations/1516653373_add_watched_event_logs.down.sql +++ b/db/migrations/1516653373_add_watched_event_logs.down.sql @@ -1 +1,2 @@ -DROP VIEW watched_event_logs; \ No newline at end of file +DROP VIEW watched_event_logs; +DROP VIEW block_stats; diff --git a/cmd/utils.go b/utils/utils.go similarity index 95% rename from cmd/utils.go rename to utils/utils.go index 03a6be65..ad091437 100644 --- a/cmd/utils.go +++ b/utils/utils.go @@ -1,4 +1,4 @@ -package cmd +package utils import ( "log" @@ -7,6 +7,8 @@ import ( "math/big" + "os" + "github.com/8thlight/vulcanizedb/pkg/config" "github.com/8thlight/vulcanizedb/pkg/core" "github.com/8thlight/vulcanizedb/pkg/geth" @@ -40,7 +42,8 @@ func ReadAbiFile(abiFilepath string) string { func AbsFilePath(filePath string) string { if !filepath.IsAbs(filePath) { - filePath = filepath.Join(config.ProjectRoot(), filePath) + cwd, _ := os.Getwd() + filePath = filepath.Join(cwd, filePath) } return filePath }