diff --git a/.gitignore b/.gitignore index ffe62712..55774619 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,4 @@ Vagrantfile vagrant_bootstrap.sh .vagrant test_scripts/ -vulcanize +vulcanizedb diff --git a/.travis.yml b/.travis.yml index 5b51065f..c3c9b8d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,11 +8,8 @@ addons: postgresql: "9.6" before_install: - # godo - - go get -u gopkg.in/godo.v2/cmd/godo # dep - go get -u github.com/golang/dep/cmd/dep - - dep ensure # ginkgo - go get -u github.com/onsi/ginkgo/ginkgo # migrate @@ -30,7 +27,7 @@ install: before_script: - ./scripts/setup - nohup ./scripts/start_private_blockchain db/schema.sql", cfg.Database.Name) - context.Bash(migrate) - context.Bash(dumpSchema) - }) - - p.Task("rollback", nil, func(context *do.Context) { - environment := parseEnvironment(context) - cfg := utils.LoadConfig(environment) - connectString := config.DbConnectionString(cfg.Database) - migrate := fmt.Sprintf("migrate -database '%s' -path ./db/migrations down 1", connectString) - dumpSchema := fmt.Sprintf("pg_dump -O -s %s > db/schema.sql", cfg.Database.Name) - context.Bash(migrate) - context.Bash(dumpSchema) - }) - - p.Task("showContractSummary", nil, func(context *do.Context) { - environment := parseEnvironment(context) - contractHash := context.Args.MayString("", "contract-hash", "c") - blockNumber := context.Args.MayInt(-1, "block-number", "b") - if contractHash == "" { - log.Fatalln("--contract-hash required") - } - context.Start(`go run main.go --environment={{.environment}} --contract-hash={{.contractHash}} --block-number={{.blockNumber}}`, - do.M{"environment": environment, - "contractHash": contractHash, - "blockNumber": blockNumber, - "$in": "cmd/show_contract_summary"}) - }) - -} - -func main() { - do.Godo(tasks) -} diff --git a/Makefile b/Makefile index 165ee223..43a43a5a 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ -HOST_NAME ?= $(DB_HOST_NAME) -PORT ?= $(DB_PORT) -NAME ?= $(DB_NAME) +HOST_NAME = +PORT = +NAME = CONNECT_STRING=postgresql://$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable $(MATTESMIGRATE): @@ -14,7 +14,7 @@ $(GINKGO): go get -u github.com/onsi/ginkgo/ginkgo checkdbvars: - test -n "$(HOST_NAME)" # $$HOST_NAME + test -n "$(HOST_NAME)" # $$HOST_NAME test -n "$(PORT)" # $$PORT test -n "$(NAME)" # $$NAME @@ -26,11 +26,15 @@ migrate: $(MATTESMIGRATE) checkdbvars migrate -database $(CONNECT_STRING) -path ./db/migrations up pg_dump -O -s $(CONNECT_STRING) > db/schema.sql +import: + test -n "$(NAME)" # $$NAME + psql $(NAME) < db/schema.sql + dep: $(DEP) dep ensure -build: dep - go build -o vulcanize ./commands +build: dep + go build test: $(GINKGO) ginkgo -r diff --git a/README.md b/README.md index b8dc09e3..adb9a3c8 100644 --- a/README.md +++ b/README.md @@ -2,20 +2,49 @@ [![Build Status](https://travis-ci.com/8thlight/vulcanizedb.svg?token=GKv2Y33qsFnfYgejjvYx&branch=master)](https://travis-ci.com/8thlight/vulcanizedb) -## Development Setup - ### Dependencies - Go 1.9+ - - https://github.com/golang/dep - - `go get -u github.com/golang/dep/cmd/dep` - - https://github.com/go-godo/godo - - `go get -u gopkg.in/godo.v2/cmd/godo` - Postgres 10 - Go Ethereum - https://ethereum.github.io/go-ethereum/downloads/ + +### Installation + go get github.com/8thlight/vulcanizedb -### Cloning the Repository +### Setting up the Databases + +1. Install Postgres +2. Create a superuser for yourself and make sure `psql --list` works without prompting for a password. +3. `createdb vulcanize_private` +4. `cd $GOPATH/src/github.com/8thlight/vulcanizedb` +5. Import the schema: + `psql vulcanize_private < db/schema.sql` + or run the migrations: + `make migrate HOST_NAME=localhost NAME=vulcanize_public PORT=5432` + * See below for configuring additional environments + +Adding a new migration: `./scripts/create_migration ` + +## Start syncing with postgres +1. Start geth +2. In a separate terminal start vulcanize_db + - `vulcanizedb sync --config --starting-block-number ` + + * see `environments` for example config + +## Watch specific contract events +1. Start geth +2. In a separate terminal start vulcanize_db + - `vulcanizedb sync --config --starting-block-number ` +3. Create event filter + - `vulcanizedb addFilter --config --filter-filepath ` + + * see `filters` for example filter + +## Development Setup + +### Cloning the Repository (Private repo only) 1. `git config --global url."git@github.com:".insteadOf "https://github.com/"` - By default, `go get` does not work for private GitHub repos. This will fix that. @@ -23,32 +52,14 @@ 3. `cd $GOPATH/src/github.com/8thlight/vulcanizedb` 4. `dep ensure` -### Setting up the Databases +### Creating/Using a test node -1. Install Postgres -2. Create a superuser for yourself and make sure `psql --list` works without prompting for a password. -3. `go get -u -d github.com/mattes/migrate/cli github.com/lib/pq` -4. `go build -tags 'postgres' -o /usr/local/bin/migrate github.com/mattes/migrate/cli` -5. `createdb vulcanize_private` -6. `cd $GOPATH/src/github.com/8thlight/vulcanizedb` -7. `godo migrate -- --environment=` - * See below for configuring additional environments - -Adding a new migration: `./scripts/create_migration ` - -### Creating/Using a Private Blockchain - -Syncing the public blockchain takes many hours for the initial sync and will download 20+ GB of data. -Here are some instructions for creating a private blockchain that does not depend on having a network connection. +Syncing the against the public network takes many hours for the initial sync and will download 20+ GB of data. +Here are some instructions for creating a private test node that does not depend on having a network connection. 1. Run `./scripts/setup` to create a private blockchain with a new account. * This will result in a warning. 2. Run `./scripts/start_private_blockchain`. -3. Run `godo run -- --environment=private` to start listener. - -### Connecting to the Public Blockchain - -`./scripts/start_blockchain` ### IPC File Paths @@ -57,54 +68,7 @@ The default location for Ethereum is: - `$HOME/.ethereum` for Ubuntu - `$GOPATH/src/gihub.com/8thlight/vulcanizedb/test_data_dir/geth.ipc` for private blockchain. -**Note the location of the ipc file is outputted when you connect to a blockchain. It is needed to for configuration** - -## Start Vulcanize DB -1. Start geth -2. In a separate terminal start vulcanize_db - - `godo vulcanizeDb -- --environment=` - -## Watch contract events -1. Start geth -2. In a separate terminal start vulcanize_db - - `godo vulcanizeDb -- --environment=` -3. Create event filter - - `godo watchEvent -- --environment= --filter-filepath=` - -## Running Listener - -1. Start geth -2. In a separate terminal start listener (ipcDir location) - - `godo run -- --environment=` - -## Retrieving Historical Data - -1. Start geth -2. In a separate terminal start listener (ipcDir location) - - `godo populateBlocks -- --environment= --starting-number=` - -## Retrieve Contract Attributes - -1. Add contract ABI to contracts / environment directory: -``` -vulcanizedb/ - contracts/ - public/ - .json - private/ -``` -The name of the JSON file should correspond the contract's address. - -2. Start watching the contract `godo watchContract -- --environment= --contract-hash=` -3. Request summary data `godo showContractSummary -- --environment= --contract-hash=` - -### Configuring Additional Environments - -You can create configuration files for additional environments. - - * Among other things, it will require the IPC file path - * See `environments/private.toml` for an example - * You will need to do this if you want to run a node connecting to the public blockchain +**Note the location of the ipc file is printed to the console when you start geth. It is needed to for configuration** ## Running the Tests diff --git a/commands/cmd/addFilter.go b/cmd/addFilter.go similarity index 93% rename from commands/cmd/addFilter.go rename to cmd/addFilter.go index 21318429..bfea9258 100644 --- a/commands/cmd/addFilter.go +++ b/cmd/addFilter.go @@ -14,12 +14,12 @@ import ( // addFilterCmd represents the addFilter command var addFilterCmd = &cobra.Command{ Use: "addFilter", - Short: "Adds event filter to vulcanize_db", + Short: "Adds event filter to vulcanizedb", Long: `An event filter is added to the vulcanize_db. All events matching the filter conitions will be tracked -in vulcanize. +in vulcanizedb. -vulcanize addFilter --config config.toml --filter-filepath filter.json +vulcanizedb addFilter --config config.toml --filter-filepath filter.json The event filters are expected to match the format described in the ethereum RPC wiki: diff --git a/cmd/add_filter/main.go b/cmd/add_filter/main.go deleted file mode 100644 index ac8f2655..00000000 --- a/cmd/add_filter/main.go +++ /dev/null @@ -1,38 +0,0 @@ -package main - -import ( - "encoding/json" - "flag" - "io/ioutil" - "log" - - "github.com/8thlight/vulcanizedb/pkg/filters" - "github.com/8thlight/vulcanizedb/pkg/geth" - "github.com/8thlight/vulcanizedb/utils" -) - -func main() { - environment := flag.String("environment", "", "Environment name") - filterFilePath := flag.String("filter-filepath", "", "path/to/filter.json") - - flag.Parse() - var logFilters filters.LogFilters - config := utils.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := utils.LoadPostgres(config.Database, blockchain.Node()) - absFilePath := utils.AbsFilePath(*filterFilePath) - logFilterBytes, err := ioutil.ReadFile(absFilePath) - if err != nil { - log.Fatal(err) - } - err = json.Unmarshal(logFilterBytes, &logFilters) - if err != nil { - log.Fatal(err) - } - for _, filter := range logFilters { - err = repository.AddFilter(filter) - if err != nil { - log.Fatal(err) - } - } -} diff --git a/cmd/populate_blocks/main.go b/cmd/populate_blocks/main.go deleted file mode 100644 index e367bf46..00000000 --- a/cmd/populate_blocks/main.go +++ /dev/null @@ -1,22 +0,0 @@ -package main - -import ( - "flag" - - "fmt" - - "github.com/8thlight/vulcanizedb/pkg/geth" - "github.com/8thlight/vulcanizedb/pkg/history" - "github.com/8thlight/vulcanizedb/utils" -) - -func main() { - environment := flag.String("environment", "", "Environment name") - startingBlockNumber := flag.Int("starting-number", -1, "First block to fill from") - flag.Parse() - config := utils.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := utils.LoadPostgres(config.Database, blockchain.Node()) - numberOfBlocksCreated := history.PopulateMissingBlocks(blockchain, repository, int64(*startingBlockNumber)) - fmt.Printf("Populated %d blocks", numberOfBlocksCreated) -} diff --git a/commands/cmd/root.go b/cmd/root.go similarity index 77% rename from commands/cmd/root.go rename to cmd/root.go index 69be2a30..ad5a1b9b 100644 --- a/commands/cmd/root.go +++ b/cmd/root.go @@ -15,7 +15,7 @@ var databaseConfig config.Database var ipc string var rootCmd = &cobra.Command{ - Use: "vulcanize", + Use: "vulcanizedb", PersistentPreRun: database, } @@ -39,11 +39,11 @@ func database(cmd *cobra.Command, args []string) { func init() { cobra.OnInitialize(initConfig) - rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file: (default is $HOME/.vulcanize.yaml)") - rootCmd.PersistentFlags().String("database-name", "vulcanize", "database: name") - rootCmd.PersistentFlags().Int("database-port", 5432, "database: port") - rootCmd.PersistentFlags().String("database-hostname", "localhost", "database: hostname") - rootCmd.PersistentFlags().String("client-ipcPath", "", "geth: geth.ipc file") + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "environment/public.toml", "config file location") + rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name") + rootCmd.PersistentFlags().Int("database-port", 5432, "database port") + rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname") + rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file") viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name")) viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port")) @@ -63,7 +63,7 @@ func initConfig() { } viper.AddConfigPath(home) - viper.SetConfigName(".vulcanize") + viper.SetConfigName(".vulcanizedb") } viper.AutomaticEnv() diff --git a/cmd/run/main.go b/cmd/run/main.go deleted file mode 100644 index f95d3fe4..00000000 --- a/cmd/run/main.go +++ /dev/null @@ -1,35 +0,0 @@ -package main - -import ( - "flag" - - "time" - - "os" - - "github.com/8thlight/vulcanizedb/pkg/geth" - "github.com/8thlight/vulcanizedb/pkg/history" - "github.com/8thlight/vulcanizedb/utils" -) - -const ( - pollingInterval = 7 * time.Second -) - -func main() { - environment := flag.String("environment", "", "Environment name") - flag.Parse() - - ticker := time.NewTicker(pollingInterval) - defer ticker.Stop() - - config := utils.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := utils.LoadPostgres(config.Database, blockchain.Node()) - validator := history.NewBlockValidator(blockchain, repository, 15) - - for range ticker.C { - window := validator.ValidateBlocks() - validator.Log(os.Stdout, window) - } -} diff --git a/cmd/show_contract_summary/main.go b/cmd/show_contract_summary/main.go deleted file mode 100644 index 1f03d15e..00000000 --- a/cmd/show_contract_summary/main.go +++ /dev/null @@ -1,35 +0,0 @@ -package main - -import ( - "flag" - - "log" - - "fmt" - - "strings" - - "github.com/8thlight/vulcanizedb/pkg/contract_summary" - "github.com/8thlight/vulcanizedb/pkg/geth" - "github.com/8thlight/vulcanizedb/utils" -) - -func main() { - environment := flag.String("environment", "", "Environment name") - contractHash := flag.String("contract-hash", "", "Contract hash to show summary") - _blockNumber := flag.Int64("block-number", -1, "Block number of summary") - flag.Parse() - - contractHashLowered := strings.ToLower(*contractHash) - config := utils.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := utils.LoadPostgres(config.Database, blockchain.Node()) - blockNumber := utils.RequestedBlockNumber(_blockNumber) - - contractSummary, err := contract_summary.NewSummary(blockchain, repository, contractHashLowered, blockNumber) - if err != nil { - log.Fatalln(err) - } - output := contract_summary.GenerateConsoleOutput(contractSummary) - fmt.Println(output) -} diff --git a/commands/cmd/sync.go b/cmd/sync.go similarity index 91% rename from commands/cmd/sync.go rename to cmd/sync.go index 214d0ce0..57408a42 100644 --- a/commands/cmd/sync.go +++ b/cmd/sync.go @@ -16,9 +16,9 @@ import ( // syncCmd represents the sync command var syncCmd = &cobra.Command{ Use: "sync", - Short: "Syncs vulcanize_db with local ethereum node", - Long: `Syncs vulcanize_db with local ethereum node. -vulcanize sync --startingBlockNumber 0 --config public.toml + Short: "Syncs vulcanizedb with local ethereum node", + Long: `Syncs vulcanizedb with local ethereum node. +vulcanizedb sync --startingBlockNumber 0 --config public.toml Expects ethereum node to be running and requires a .toml config: diff --git a/cmd/vulcanize_db/main.go b/cmd/vulcanize_db/main.go deleted file mode 100644 index 366f8615..00000000 --- a/cmd/vulcanize_db/main.go +++ /dev/null @@ -1,53 +0,0 @@ -package main - -import ( - "flag" - - "time" - - "os" - - "github.com/8thlight/vulcanizedb/pkg/core" - "github.com/8thlight/vulcanizedb/pkg/geth" - "github.com/8thlight/vulcanizedb/pkg/history" - "github.com/8thlight/vulcanizedb/pkg/repositories" - "github.com/8thlight/vulcanizedb/utils" -) - -const ( - pollingInterval = 7 * time.Second -) - -func backFillAllBlocks(blockchain core.Blockchain, repository repositories.Postgres, missingBlocksPopulated chan int, startingBlockNumber int64) { - go func() { - missingBlocksPopulated <- history.PopulateMissingBlocks(blockchain, repository, startingBlockNumber) - }() -} - -func main() { - environment := flag.String("environment", "", "Environment name") - startingBlockNumber := flag.Int("starting-number", 0, "First block to fill from") - flag.Parse() - - ticker := time.NewTicker(pollingInterval) - defer ticker.Stop() - - config := utils.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := utils.LoadPostgres(config.Database, blockchain.Node()) - validator := history.NewBlockValidator(blockchain, repository, 15) - - missingBlocksPopulated := make(chan int) - _startingBlockNumber := int64(*startingBlockNumber) - go backFillAllBlocks(blockchain, repository, missingBlocksPopulated, _startingBlockNumber) - - for { - select { - case <-ticker.C: - window := validator.ValidateBlocks() - validator.Log(os.Stdout, window) - case <-missingBlocksPopulated: - go backFillAllBlocks(blockchain, repository, missingBlocksPopulated, _startingBlockNumber) - } - } -} diff --git a/cmd/watch_contract/main.go b/cmd/watch_contract/main.go deleted file mode 100644 index 1a02b52d..00000000 --- a/cmd/watch_contract/main.go +++ /dev/null @@ -1,31 +0,0 @@ -package main - -import ( - "flag" - - "strings" - - "github.com/8thlight/vulcanizedb/pkg/core" - "github.com/8thlight/vulcanizedb/pkg/geth" - "github.com/8thlight/vulcanizedb/utils" -) - -func main() { - environment := flag.String("environment", "", "Environment name") - contractHash := flag.String("contract-hash", "", "contract-hash=x1234") - abiFilepath := flag.String("abi-filepath", "", "path/to/abifile.json") - network := flag.String("network", "", "ropsten") - - flag.Parse() - contractHashLowered := strings.ToLower(*contractHash) - - contractAbiString := utils.GetAbi(*abiFilepath, contractHashLowered, *network) - config := utils.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := utils.LoadPostgres(config.Database, blockchain.Node()) - watchedContract := core.Contract{ - Abi: contractAbiString, - Hash: contractHashLowered, - } - repository.CreateContract(watchedContract) -} diff --git a/commands/README.md b/commands/README.md deleted file mode 100644 index 5349be62..00000000 --- a/commands/README.md +++ /dev/null @@ -1,90 +0,0 @@ -# Vulcanize DB - -[![Build Status](https://travis-ci.com/8thlight/vulcanizedb.svg?token=GKv2Y33qsFnfYgejjvYx&branch=master)](https://travis-ci.com/8thlight/vulcanizedb) - -## Development Setup - -### Dependencies - - - Go 1.9+ - - Postgres 10 - - Go Ethereum - - https://ethereum.github.io/go-ethereum/downloads/ - -### Cloning the Repository - -1. `git config --global url."git@github.com:".insteadOf "https://github.com/"` - - By default, `go get` does not work for private GitHub repos. This will fix that. -2. `go get github.com/8thlight/vulcanizedb` -3. `cd $GOPATH/src/github.com/8thlight/vulcanizedb` -4. `dep ensure` - -### Setting up the Databases - -1. Install Postgres -2. Create a superuser for yourself and make sure `psql --list` works without prompting for a password. -3. `createdb vulcanize_private` -4. `cd $GOPATH/src/github.com/8thlight/vulcanizedb` -5. `HOST_NAME=localhost NAME=vulcanize_public PORT=5432 make migrate` - * See below for configuring additional environments - -Adding a new migration: `./scripts/create_migration ` - -### Building -1. `make build` - -### Creating/Using a Private Blockchain - -Syncing the public blockchain takes many hours for the initial sync and will download 20+ GB of data. -Here are some instructions for creating a private blockchain that does not depend on having a network connection. - -1. Run `./scripts/setup` to create a private blockchain with a new account. - * This will result in a warning. -2. Run `./scripts/start_private_blockchain`. -3. Run `godo run -- --environment=private` to start listener. - -### Connecting to the Public Blockchain - -`./scripts/start_blockchain` - -### IPC File Paths - -The default location for Ethereum is: - - `$HOME/Library/Ethereum` for Mac - - `$HOME/.ethereum` for Ubuntu - - `$GOPATH/src/gihub.com/8thlight/vulcanizedb/test_data_dir/geth.ipc` for private blockchain. - -**Note the location of the ipc file is outputted when you connect to geth. It is needed to for configuration** - -## Start Vulcanize DB -1. Start geth -2. In a separate terminal start vulcanize_db - - `vulcanize sync --config --starting-block-number ` - -## Watch contract events -1. Start geth -2. In a separate terminal start vulcanize_db - - `vulcanize sync --config --starting-block-number ` -3. Create event filter - - `vulcanize addFilter --config --filter-filepath ` - -### Configuring Additional Environments - -You can create configuration files for additional environments. - - * Among other things, it will require the IPC file path - * See `environments/private.toml` for an example - * You will need to do this if you want to run a node connecting to the public blockchain - -## Running the Tests - -### Unit Tests - -1. `go test ./pkg/...` - -### Integration Test - -In order to run the integration tests, you will need to run them against a real blockchain. At the moment the integration tests require [Geth v1.7.2](https://ethereum.github.io/go-ethereum/downloads/) as they depend on the `--dev` mode, which changed in v1.7.3 - -1. Run `./scripts/start_private_blockchain` as a separate process. -2. `go test ./...` to run all tests. diff --git a/db/schema.sql b/db/schema.sql index 9ba46900..eb98c994 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -404,6 +404,14 @@ ALTER TABLE ONLY watched_contracts ADD CONSTRAINT contract_hash_uc UNIQUE (contract_hash); +-- +-- Name: logs log_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY logs + ADD CONSTRAINT log_uc UNIQUE (block_number, index); + + -- -- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- diff --git a/commands/main.go b/main.go similarity index 54% rename from commands/main.go rename to main.go index 50e90e3b..8bdcc8e2 100644 --- a/commands/main.go +++ b/main.go @@ -1,7 +1,7 @@ package main import ( - "github.com/8thlight/vulcanizedb/commands/cmd" + "github.com/8thlight/vulcanizedb/cmd" ) func main() {