diff --git a/.gitignore b/.gitignore index ac6ee651..55774619 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ environments/*.toml Vagrantfile vagrant_bootstrap.sh .vagrant +test_scripts/ +vulcanizedb diff --git a/.travis.yml b/.travis.yml index 5b51065f..9cac17ce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,12 +7,11 @@ services: addons: postgresql: "9.6" +go_import_path: github.com/vulcanize/vulcanizedb + before_install: - # godo - - go get -u gopkg.in/godo.v2/cmd/godo # dep - go get -u github.com/golang/dep/cmd/dep - - dep ensure # ginkgo - go get -u github.com/onsi/ginkgo/ginkgo # migrate @@ -30,7 +29,7 @@ install: before_script: - ./scripts/setup - nohup ./scripts/start_private_blockchain db/schema.sql + +migrate: $(MATTESMIGRATE) checkdbvars + migrate -database $(CONNECT_STRING) -path ./db/migrations up + pg_dump -O -s $(CONNECT_STRING) > db/schema.sql + +import: + test -n "$(NAME)" # $$NAME + psql $(NAME) < db/schema.sql + +dep: $(DEP) + dep ensure + +build: dep + go build + +test: $(GINKGO) + ginkgo -r + +createprivate: + #!/bin/bash + echo "Deleting test blockchain" + rm -rf test_data_dir + echo "Creating test blockchain with a new account" + mkdir test_data_dir + geth --dev --datadir test_data_dir --password .private_blockchain_password account new + +startprivate: createprivate + geth --datadir test_data_dir --dev --nodiscover --mine --minerthreads 1 --maxpeers 0 --verbosity 3 --unlock 0 --password .private_blockchain_password --rpc diff --git a/README.md b/README.md index ae38cd45..7a78ccce 100644 --- a/README.md +++ b/README.md @@ -1,109 +1,54 @@ # Vulcanize DB -[![Build Status](https://travis-ci.com/vulcanize/vulcanizedb.svg?token=GKv2Y33qsFnfYgejjvYx&branch=master)](https://travis-ci.com/vulcanize/vulcanizedb) - -## Development Setup +[![Build Status](https://travis-ci.com/8thlight/vulcanizedb.svg?token=3psFYN2533rYjhRbvjte&branch=master)](https://travis-ci.com/8thlight/vulcanizedb) ### Dependencies - Go 1.9+ - - https://github.com/golang/dep - - `go get -u github.com/golang/dep/cmd/dep` - - https://github.com/go-godo/godo - - `go get -u gopkg.in/godo.v2/cmd/godo` - Postgres 10 - Go Ethereum - https://ethereum.github.io/go-ethereum/downloads/ + +### Installation + go get github.com/vulcanize/vulcanizedb -### Cloning the Repository - -1. `git config --global url."git@github.com:".insteadOf "https://github.com/"` - - By default, `go get` does not work for private GitHub repos. This will fix that. -2. `go get github.com/vulcanize/vulcanizedb` -3. `cd $GOPATH/src/github.com/vulcanize/vulcanizedb` -4. `dep ensure` - ### Setting up the Databases 1. Install Postgres 2. Create a superuser for yourself and make sure `psql --list` works without prompting for a password. -3. `go get -u -d github.com/mattes/migrate/cli github.com/lib/pq` -4. `go build -tags 'postgres' -o /usr/local/bin/migrate github.com/mattes/migrate/cli` -5. `createdb vulcanize_private` -6. `cd $GOPATH/src/github.com/vulcanize/vulcanizedb` -7. `godo migrate -- --environment=` +3. `createdb vulcanize_private` +4. `cd $GOPATH/src/github.com/vulcanize/vulcanizedb` +5. Import the schema: `psql vulcanize_private < db/schema.sql` + + or run the migrations: `make migrate HOST_NAME=localhost NAME=vulcanize_public PORT=5432` * See below for configuring additional environments - -Adding a new migration: `./scripts/create_migration ` - -### Creating/Using a Private Blockchain - -Syncing the public blockchain takes many hours for the initial sync and will download 20+ GB of data. -Here are some instructions for creating a private blockchain that does not depend on having a network connection. - -1. Run `./scripts/setup` to create a private blockchain with a new account. - * This will result in a warning. -2. Run `./scripts/start_private_blockchain`. -3. Run `godo run -- --environment=private` to start listener. - -### Connecting to the Public Blockchain - -`./scripts/start_blockchain` - + ### IPC File Paths The default location for Ethereum is: - `$HOME/Library/Ethereum` for Mac - `$HOME/.ethereum` for Ubuntu - - `$GOPATH/src/gihub.com/vulcanize/vulcanizedb/test_data_dir/geth.ipc` for private blockchain. + - `$GOPATH/src/gihub.com/vulcanize/vulcanizedb/test_data_dir/geth.ipc` for private node. -**Note the location of the ipc file is outputted when you connect to a blockchain. It is needed to for configuration** +**Note the location of the ipc file is printed to the console when you start geth. It is needed to for configuration** -## Start Vulcanize DB -1. Start a blockchain. +## Start syncing with postgres +1. Start geth node (**if fast syncing wait for geth to finsh initial sync**) 2. In a separate terminal start vulcanize_db - - `godo vulcanizeDb -- --environment=` - -## Running Listener - -1. Start a blockchain. -2. In a separate terminal start listener (ipcDir location) - - `godo run -- --environment=` + - `vulcanizedb sync --config --starting-block-number ` -## Retrieving Historical Data - -1. Start a blockchain. -2. In a separate terminal start listener (ipcDir location) - - `godo populateBlocks -- --environment= --starting-number=` - -## Retrieve Contract Attributes - -1. Add contract ABI to contracts / environment directory: -``` -vulcanizedb/ - contracts/ - public/ - .json - private/ -``` -The name of the JSON file should correspond the contract's address. -2. Start watching the contract `godo watchContract -- --environment= --contract-hash=` -3. Request summary data `godo showContractSummary -- --environment= --contract-hash=` - - -## Retrieving Contract Logs - -1. Get the logs for a specific contract - - `godo getLogs -- --environment= --contract-hash=` - -### Configuring Additional Environments - -You can create configuration files for additional environments. - - * Among other things, it will require the IPC file path - * See `environments/private.toml` for an example - * You will need to do this if you want to run a node connecting to the public blockchain + * see `./environments` for example config +## Watch specific events +1. Start geth +2. In a separate terminal start vulcanize_db + - `vulcanizedb sync --config --starting-block-number ` +3. Create event filter + - `vulcanizedb addFilter --config --filter-filepath ` + * see `./filters` for example filter +4. The filters are tracked in the `log_filters` table and the filtered events +will show up in the `watched_log_events` view + ## Running the Tests ### Unit Tests @@ -112,7 +57,12 @@ You can create configuration files for additional environments. ### Integration Test -In order to run the integration tests, you will need to run them against a real blockchain. At the moment the integration tests require [Geth v1.7.2](https://ethereum.github.io/go-ethereum/downloads/) as they depend on the `--dev` mode, which changed in v1.7.3 +In order to run the integration tests, you will need to run them against a real node. At the moment the integration tests require [Geth v1.7.2](https://ethereum.github.io/go-ethereum/downloads/) as they depend on the `--dev` mode, which changed in v1.7.3 -1. Run `./scripts/start_private_blockchain` as a separate process. -2. `go test ./...` to run all tests. \ No newline at end of file +1. Run `make startprivate` in a separate terminal +2. Setup a test database and import the schema: + + `createdb vulcanize_private` + + `psql vulcanize_private < db/schema.sql` +3. `go test ./...` to run all tests. diff --git a/cmd/addFilter.go b/cmd/addFilter.go new file mode 100644 index 00000000..2b83c5a6 --- /dev/null +++ b/cmd/addFilter.go @@ -0,0 +1,75 @@ +package cmd + +import ( + "encoding/json" + "io/ioutil" + "log" + + "github.com/vulcanize/vulcanizedb/pkg/filters" + "github.com/vulcanize/vulcanizedb/pkg/geth" + "github.com/vulcanize/vulcanizedb/utils" + "github.com/spf13/cobra" +) + +// addFilterCmd represents the addFilter command +var addFilterCmd = &cobra.Command{ + Use: "addFilter", + Short: "Adds event filter to vulcanizedb", + Long: `An event filter is added to the vulcanize_db. +All events matching the filter conitions will be tracked +in vulcanizedb. + +vulcanizedb addFilter --config config.toml --filter-filepath filter.json + +The event filters are expected to match +the format described in the ethereum RPC wiki: + +https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_newfilter + +[{ + "fromBlock": "0x1", + "toBlock": "0x2", + "address": "0x8888f1f195afa192cfee860698584c030f4c9db1", + "topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + null, + "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "0x0000000000000000000000000aff3454fce5edbc8cca8697c15331677e6ebccc"] +}] +`, + Run: func(cmd *cobra.Command, args []string) { + addFilter() + }, +} + +var filterFilepath string + +func init() { + rootCmd.AddCommand(addFilterCmd) + + addFilterCmd.PersistentFlags().StringVar(&filterFilepath, "filter-filepath", "", "path/to/filter.json") + addFilterCmd.MarkFlagRequired("filter-filepath") +} + +func addFilter() { + if filterFilepath == "" { + log.Fatal("filter-filepath required") + } + var logFilters filters.LogFilters + blockchain := geth.NewBlockchain(ipc) + repository := utils.LoadPostgres(databaseConfig, blockchain.Node()) + absFilePath := utils.AbsFilePath(filterFilepath) + logFilterBytes, err := ioutil.ReadFile(absFilePath) + if err != nil { + log.Fatal(err) + } + err = json.Unmarshal(logFilterBytes, &logFilters) + if err != nil { + log.Fatal(err) + } + for _, filter := range logFilters { + err = repository.AddFilter(filter) + if err != nil { + log.Fatal(err) + } + } +} diff --git a/cmd/get_logs/main.go b/cmd/get_logs/main.go deleted file mode 100644 index ca79bd25..00000000 --- a/cmd/get_logs/main.go +++ /dev/null @@ -1,71 +0,0 @@ -package main - -import ( - "log" - - "flag" - - "math/big" - - "time" - - "github.com/vulcanize/vulcanizedb/cmd" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/geth" -) - -func min(a, b int64) int64 { - if a < b { - return a - } - return b -} - -const ( - windowSize = 24 - pollingInterval = 10 * time.Second -) - -func main() { - environment := flag.String("environment", "", "Environment name") - contractHash := flag.String("contract-hash", "", "Contract hash to show summary") - ticker := time.NewTicker(pollingInterval) - defer ticker.Stop() - - flag.Parse() - - config := cmd.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) - - lastBlockNumber := blockchain.LastBlock().Int64() - stepSize := int64(1000) - - go func() { - for i := int64(0); i < lastBlockNumber; i = min(i+stepSize, lastBlockNumber) { - logs, err := blockchain.GetLogs(core.Contract{Hash: *contractHash}, big.NewInt(i), big.NewInt(i+stepSize)) - log.Println("Backfilling Logs:", i) - if err != nil { - log.Println(err) - } - repository.CreateLogs(logs) - } - }() - - done := make(chan struct{}) - go func() { done <- struct{}{} }() - for range ticker.C { - select { - case <-done: - go func() { - z := &big.Int{} - z.Sub(blockchain.LastBlock(), big.NewInt(25)) - log.Printf("Logs Window: %d - %d", z.Int64(), blockchain.LastBlock().Int64()) - logs, _ := blockchain.GetLogs(core.Contract{Hash: *contractHash}, z, blockchain.LastBlock()) - repository.CreateLogs(logs) - done <- struct{}{} - }() - default: - } - } -} diff --git a/cmd/populate_blocks/main.go b/cmd/populate_blocks/main.go deleted file mode 100644 index f402b23d..00000000 --- a/cmd/populate_blocks/main.go +++ /dev/null @@ -1,22 +0,0 @@ -package main - -import ( - "flag" - - "fmt" - - "github.com/vulcanize/vulcanizedb/cmd" - "github.com/vulcanize/vulcanizedb/pkg/geth" - "github.com/vulcanize/vulcanizedb/pkg/history" -) - -func main() { - environment := flag.String("environment", "", "Environment name") - startingBlockNumber := flag.Int("starting-number", -1, "First block to fill from") - flag.Parse() - config := cmd.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) - numberOfBlocksCreated := history.PopulateMissingBlocks(blockchain, repository, int64(*startingBlockNumber)) - fmt.Printf("Populated %d blocks", numberOfBlocksCreated) -} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 00000000..5fdec651 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,74 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/mitchellh/go-homedir" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var cfgFile string +var databaseConfig config.Database +var ipc string + +var rootCmd = &cobra.Command{ + Use: "vulcanizedb", + PersistentPreRun: database, +} + +func Execute() { + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func database(cmd *cobra.Command, args []string) { + ipc = viper.GetString("client.ipcpath") + databaseConfig = config.Database{ + Name: viper.GetString("database.name"), + Hostname: viper.GetString("database.hostname"), + Port: viper.GetInt("database.port"), + } + viper.Set("database.config", databaseConfig) +} + +func init() { + cobra.OnInitialize(initConfig) + + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "environment/public.toml", "config file location") + rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name") + rootCmd.PersistentFlags().Int("database-port", 5432, "database port") + rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname") + rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file") + + viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name")) + viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port")) + viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname")) + viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath")) + +} + +func initConfig() { + if cfgFile != "" { + viper.SetConfigFile(cfgFile) + } else { + home, err := homedir.Dir() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + viper.AddConfigPath(home) + viper.SetConfigName(".vulcanizedb") + } + + viper.AutomaticEnv() + + if err := viper.ReadInConfig(); err == nil { + fmt.Printf("Using config file: %s\n\n", viper.ConfigFileUsed()) + } +} diff --git a/cmd/run/main.go b/cmd/run/main.go deleted file mode 100644 index e9c5ae8d..00000000 --- a/cmd/run/main.go +++ /dev/null @@ -1,34 +0,0 @@ -package main - -import ( - "flag" - - "time" - - "os" - - "github.com/vulcanize/vulcanizedb/cmd" - "github.com/vulcanize/vulcanizedb/pkg/geth" - "github.com/vulcanize/vulcanizedb/pkg/history" -) - -const ( - pollingInterval = 7 * time.Second -) - -func main() { - ticker := time.NewTicker(pollingInterval) - defer ticker.Stop() - - environment := flag.String("environment", "", "Environment name") - flag.Parse() - config := cmd.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) - validator := history.NewBlockValidator(blockchain, repository, 15) - - for range ticker.C { - window := validator.ValidateBlocks() - validator.Log(os.Stdout, window) - } -} diff --git a/cmd/show_contract_summary/main.go b/cmd/show_contract_summary/main.go deleted file mode 100644 index 24e83cf2..00000000 --- a/cmd/show_contract_summary/main.go +++ /dev/null @@ -1,31 +0,0 @@ -package main - -import ( - "flag" - - "log" - - "fmt" - - "github.com/vulcanize/vulcanizedb/cmd" - "github.com/vulcanize/vulcanizedb/pkg/contract_summary" - "github.com/vulcanize/vulcanizedb/pkg/geth" -) - -func main() { - environment := flag.String("environment", "", "Environment name") - contractHash := flag.String("contract-hash", "", "Contract hash to show summary") - _blockNumber := flag.Int64("block-number", -1, "Block number of summary") - flag.Parse() - config := cmd.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) - blockNumber := cmd.RequestedBlockNumber(_blockNumber) - - contractSummary, err := contract_summary.NewSummary(blockchain, repository, *contractHash, blockNumber) - if err != nil { - log.Fatalln(err) - } - output := contract_summary.GenerateConsoleOutput(contractSummary) - fmt.Println(output) -} diff --git a/cmd/sync.go b/cmd/sync.go new file mode 100644 index 00000000..748f23ce --- /dev/null +++ b/cmd/sync.go @@ -0,0 +1,81 @@ +package cmd + +import ( + "os" + + "time" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/geth" + "github.com/vulcanize/vulcanizedb/pkg/history" + "github.com/vulcanize/vulcanizedb/pkg/repositories" + "github.com/vulcanize/vulcanizedb/utils" + "github.com/spf13/cobra" + "log" +) + +// syncCmd represents the sync command +var syncCmd = &cobra.Command{ + Use: "sync", + Short: "Syncs vulcanizedb with local ethereum node", + Long: `Syncs vulcanizedb with local ethereum node. +vulcanizedb sync --startingBlockNumber 0 --config public.toml + +Expects ethereum node to be running and requires a .toml config: + + [database] + name = "vulcanize_public" + hostname = "localhost" + port = 5432 + + [client] + ipcPath = "/Users/mattkrump/Library/Ethereum/geth.ipc" +`, + Run: func(cmd *cobra.Command, args []string) { + sync() + }, +} + +const ( + pollingInterval = 7 * time.Second +) + +var startingBlockNumber int + +func init() { + rootCmd.AddCommand(syncCmd) + + syncCmd.Flags().IntVarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from") +} + +func backFillAllBlocks(blockchain core.Blockchain, repository repositories.Postgres, missingBlocksPopulated chan int, startingBlockNumber int64) { + go func() { + missingBlocksPopulated <- history.PopulateMissingBlocks(blockchain, repository, startingBlockNumber) + }() +} + +func sync() { + ticker := time.NewTicker(pollingInterval) + defer ticker.Stop() + + blockchain := geth.NewBlockchain(ipc) + if blockchain.LastBlock().Int64() == 0 { + log.Fatal("geth initial: state sync not finished") + } + repository := utils.LoadPostgres(databaseConfig, blockchain.Node()) + validator := history.NewBlockValidator(blockchain, repository, 15) + + missingBlocksPopulated := make(chan int) + _startingBlockNumber := int64(startingBlockNumber) + go backFillAllBlocks(blockchain, repository, missingBlocksPopulated, _startingBlockNumber) + + for { + select { + case <-ticker.C: + window := validator.ValidateBlocks() + validator.Log(os.Stdout, window) + case <-missingBlocksPopulated: + go backFillAllBlocks(blockchain, repository, missingBlocksPopulated, _startingBlockNumber) + } + } +} diff --git a/cmd/vulcanize_db/main.go b/cmd/vulcanize_db/main.go deleted file mode 100644 index 2cf8cb42..00000000 --- a/cmd/vulcanize_db/main.go +++ /dev/null @@ -1,50 +0,0 @@ -package main - -import ( - "flag" - - "time" - - "os" - - "github.com/vulcanize/vulcanizedb/cmd" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/geth" - "github.com/vulcanize/vulcanizedb/pkg/history" - "github.com/vulcanize/vulcanizedb/pkg/repositories" -) - -const ( - pollingInterval = 7 * time.Second -) - -func backFillAllBlocks(blockchain core.Blockchain, repository repositories.Postgres, missingBlocksPopulated chan int) { - go func() { - missingBlocksPopulated <- history.PopulateMissingBlocks(blockchain, repository, 0) - }() -} - -func main() { - ticker := time.NewTicker(pollingInterval) - defer ticker.Stop() - - environment := flag.String("environment", "", "Environment name") - flag.Parse() - config := cmd.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) - validator := history.NewBlockValidator(blockchain, repository, 15) - - missingBlocksPopulated := make(chan int) - go backFillAllBlocks(blockchain, repository, missingBlocksPopulated) - - for { - select { - case <-ticker.C: - window := validator.ValidateBlocks() - validator.Log(os.Stdout, window) - case <-missingBlocksPopulated: - go backFillAllBlocks(blockchain, repository, missingBlocksPopulated) - } - } -} diff --git a/cmd/watch_contract/main.go b/cmd/watch_contract/main.go deleted file mode 100644 index 4b1451e8..00000000 --- a/cmd/watch_contract/main.go +++ /dev/null @@ -1,26 +0,0 @@ -package main - -import ( - "flag" - - "github.com/vulcanize/vulcanizedb/cmd" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/geth" -) - -func main() { - environment := flag.String("environment", "", "Environment name") - contractHash := flag.String("contract-hash", "", "contract-hash=x1234") - abiFilepath := flag.String("abi-filepath", "", "path/to/abifile.json") - flag.Parse() - - contractAbiString := cmd.GetAbi(*abiFilepath, *contractHash) - config := cmd.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) - watchedContract := core.Contract{ - Abi: contractAbiString, - Hash: *contractHash, - } - repository.CreateContract(watchedContract) -} diff --git a/db/migrations/1515613181_add_fields_to_node.down.sql b/db/migrations/1515613181_add_fields_to_node.down.sql new file mode 100644 index 00000000..66ffb290 --- /dev/null +++ b/db/migrations/1515613181_add_fields_to_node.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE nodes + DROP COLUMN node_id, + DROP COLUMN client_name; \ No newline at end of file diff --git a/db/migrations/1515613181_add_fields_to_node.up.sql b/db/migrations/1515613181_add_fields_to_node.up.sql new file mode 100644 index 00000000..5396e26a --- /dev/null +++ b/db/migrations/1515613181_add_fields_to_node.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE nodes + ADD COLUMN node_id VARCHAR(128), + ADD COLUMN client_name VARCHAR; \ No newline at end of file diff --git a/db/migrations/1515613715_update_node_index.down.sql b/db/migrations/1515613715_update_node_index.down.sql new file mode 100644 index 00000000..5c252c4b --- /dev/null +++ b/db/migrations/1515613715_update_node_index.down.sql @@ -0,0 +1,9 @@ +BEGIN; + +ALTER TABLE nodes + DROP CONSTRAINT node_uc; + +ALTER TABLE nodes + ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id); + +COMMIT; diff --git a/db/migrations/1515613715_update_node_index.up.sql b/db/migrations/1515613715_update_node_index.up.sql new file mode 100644 index 00000000..6abf9347 --- /dev/null +++ b/db/migrations/1515613715_update_node_index.up.sql @@ -0,0 +1,9 @@ +BEGIN; + +ALTER TABLE nodes + DROP CONSTRAINT node_uc; + +ALTER TABLE nodes + ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id); + +COMMIT; \ No newline at end of file diff --git a/db/migrations/1516050071_add_log_fk_constraint.down.sql b/db/migrations/1516050071_add_log_fk_constraint.down.sql new file mode 100644 index 00000000..55779b51 --- /dev/null +++ b/db/migrations/1516050071_add_log_fk_constraint.down.sql @@ -0,0 +1,12 @@ +BEGIN; + +ALTER TABLE logs + DROP CONSTRAINT receipts_fk; + +ALTER TABLE logs + DROP COLUMN receipt_id; + +ALTER TABLE logs + ADD CONSTRAINT log_uc UNIQUE (block_number, index); + +COMMIT; \ No newline at end of file diff --git a/db/migrations/1516050071_add_log_fk_constraint.up.sql b/db/migrations/1516050071_add_log_fk_constraint.up.sql new file mode 100644 index 00000000..60733b71 --- /dev/null +++ b/db/migrations/1516050071_add_log_fk_constraint.up.sql @@ -0,0 +1,14 @@ +BEGIN; +ALTER TABLE logs + DROP CONSTRAINT log_uc; + +ALTER TABLE logs + ADD COLUMN receipt_id INT; + +ALTER TABLE logs + ADD CONSTRAINT receipts_fk +FOREIGN KEY (receipt_id) +REFERENCES receipts (id) +ON DELETE CASCADE; + +COMMIT; \ No newline at end of file diff --git a/db/migrations/1516648743_add_log_filters.down.sql b/db/migrations/1516648743_add_log_filters.down.sql new file mode 100644 index 00000000..a28adeee --- /dev/null +++ b/db/migrations/1516648743_add_log_filters.down.sql @@ -0,0 +1 @@ +DROP TABLE log_filters; \ No newline at end of file diff --git a/db/migrations/1516648743_add_log_filters.up.sql b/db/migrations/1516648743_add_log_filters.up.sql new file mode 100644 index 00000000..100a5718 --- /dev/null +++ b/db/migrations/1516648743_add_log_filters.up.sql @@ -0,0 +1,12 @@ +CREATE TABLE log_filters ( + id SERIAL, + name VARCHAR NOT NULL CHECK (name <> ''), + from_block BIGINT CHECK (from_block >= 0), + to_block BIGINT CHECK (from_block >= 0), + address VARCHAR(66), + topic0 VARCHAR(66), + topic1 VARCHAR(66), + topic2 VARCHAR(66), + topic3 VARCHAR(66), + CONSTRAINT name_uc UNIQUE (name) +); \ No newline at end of file diff --git a/db/migrations/1516653373_add_watched_event_logs.down.sql b/db/migrations/1516653373_add_watched_event_logs.down.sql new file mode 100644 index 00000000..294d7bfe --- /dev/null +++ b/db/migrations/1516653373_add_watched_event_logs.down.sql @@ -0,0 +1,2 @@ +DROP VIEW watched_event_logs; +DROP VIEW block_stats; diff --git a/db/migrations/1516653373_add_watched_event_logs.up.sql b/db/migrations/1516653373_add_watched_event_logs.up.sql new file mode 100644 index 00000000..3bba916f --- /dev/null +++ b/db/migrations/1516653373_add_watched_event_logs.up.sql @@ -0,0 +1,29 @@ +CREATE VIEW block_stats AS + SELECT + max(block_number) AS max_block, + min(block_number) AS min_block + FROM logs; + +CREATE VIEW watched_event_logs AS + SELECT + log_filters.name, + logs.id, + block_number, + logs.address, + tx_hash, + index, + logs.topic0, + logs.topic1, + logs.topic2, + logs.topic3, + data, + receipt_id + FROM log_filters + CROSS JOIN block_stats + JOIN logs ON logs.address = log_filters.address + AND logs.block_number >= coalesce(log_filters.from_block, block_stats.min_block) + AND logs.block_number <= coalesce(log_filters.to_block, block_stats.max_block) + WHERE (log_filters.topic0 = logs.topic0 OR log_filters.topic0 ISNULL) + AND (log_filters.topic1 = logs.topic1 OR log_filters.topic1 ISNULL) + AND (log_filters.topic2 = logs.topic2 OR log_filters.topic2 ISNULL) + AND (log_filters.topic3 = logs.topic3 OR log_filters.topic3 ISNULL); \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql new file mode 100644 index 00000000..9ba46900 --- /dev/null +++ b/db/schema.sql @@ -0,0 +1,556 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 10.1 +-- Dumped by pg_dump version 10.1 + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; + + +-- +-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; + + +SET search_path = public, pg_catalog; + +SET default_tablespace = ''; + +SET default_with_oids = false; + +-- +-- Name: logs; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE logs ( + id integer NOT NULL, + block_number bigint, + address character varying(66), + tx_hash character varying(66), + index bigint, + topic0 character varying(66), + topic1 character varying(66), + topic2 character varying(66), + topic3 character varying(66), + data text, + receipt_id integer +); + + +-- +-- Name: block_stats; Type: VIEW; Schema: public; Owner: - +-- + +CREATE VIEW block_stats AS + SELECT max(logs.block_number) AS max_block, + min(logs.block_number) AS min_block + FROM logs; + + +-- +-- Name: blocks; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE blocks ( + block_number bigint, + block_gaslimit double precision, + block_gasused double precision, + block_time double precision, + id integer NOT NULL, + block_difficulty bigint, + block_hash character varying(66), + block_nonce character varying(20), + block_parenthash character varying(66), + block_size bigint, + uncle_hash character varying(66), + node_id integer NOT NULL, + is_final boolean, + block_miner character varying(42), + block_extra_data character varying, + block_reward numeric, + block_uncles_reward numeric +); + + +-- +-- Name: blocks_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE blocks_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: blocks_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE blocks_id_seq OWNED BY blocks.id; + + +-- +-- Name: log_filters; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE log_filters ( + id integer NOT NULL, + name character varying NOT NULL, + from_block bigint, + to_block bigint, + address character varying(66), + topic0 character varying(66), + topic1 character varying(66), + topic2 character varying(66), + topic3 character varying(66), + CONSTRAINT log_filters_from_block_check CHECK ((from_block >= 0)), + CONSTRAINT log_filters_from_block_check1 CHECK ((from_block >= 0)), + CONSTRAINT log_filters_name_check CHECK (((name)::text <> ''::text)) +); + + +-- +-- Name: log_filters_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE log_filters_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: log_filters_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE log_filters_id_seq OWNED BY log_filters.id; + + +-- +-- Name: logs_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE logs_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE logs_id_seq OWNED BY logs.id; + + +-- +-- Name: nodes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE nodes ( + id integer NOT NULL, + genesis_block character varying(66), + network_id numeric, + node_id character varying(128), + client_name character varying +); + + +-- +-- Name: nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE nodes_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: nodes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE nodes_id_seq OWNED BY nodes.id; + + +-- +-- Name: receipts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE receipts ( + id integer NOT NULL, + transaction_id integer NOT NULL, + contract_address character varying(42), + cumulative_gas_used numeric, + gas_used numeric, + state_root character varying(66), + status integer, + tx_hash character varying(66) +); + + +-- +-- Name: receipts_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE receipts_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: receipts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE receipts_id_seq OWNED BY receipts.id; + + +-- +-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE schema_migrations ( + version bigint NOT NULL, + dirty boolean NOT NULL +); + + +-- +-- Name: transactions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE transactions ( + id integer NOT NULL, + tx_hash character varying(66), + tx_nonce numeric, + tx_to character varying(66), + tx_gaslimit numeric, + tx_gasprice numeric, + tx_value numeric, + block_id integer NOT NULL, + tx_from character varying(66), + tx_input_data character varying +); + + +-- +-- Name: transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE transactions_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE transactions_id_seq OWNED BY transactions.id; + + +-- +-- Name: watched_contracts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE watched_contracts ( + contract_id integer NOT NULL, + contract_hash character varying(66), + contract_abi json +); + + +-- +-- Name: watched_contracts_contract_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE watched_contracts_contract_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: watched_contracts_contract_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE watched_contracts_contract_id_seq OWNED BY watched_contracts.contract_id; + + +-- +-- Name: watched_event_logs; Type: VIEW; Schema: public; Owner: - +-- + +CREATE VIEW watched_event_logs AS + SELECT log_filters.name, + logs.id, + logs.block_number, + logs.address, + logs.tx_hash, + logs.index, + logs.topic0, + logs.topic1, + logs.topic2, + logs.topic3, + logs.data, + logs.receipt_id + FROM ((log_filters + CROSS JOIN block_stats) + JOIN logs ON ((((logs.address)::text = (log_filters.address)::text) AND (logs.block_number >= COALESCE(log_filters.from_block, block_stats.min_block)) AND (logs.block_number <= COALESCE(log_filters.to_block, block_stats.max_block))))) + WHERE ((((log_filters.topic0)::text = (logs.topic0)::text) OR (log_filters.topic0 IS NULL)) AND (((log_filters.topic1)::text = (logs.topic1)::text) OR (log_filters.topic1 IS NULL)) AND (((log_filters.topic2)::text = (logs.topic2)::text) OR (log_filters.topic2 IS NULL)) AND (((log_filters.topic3)::text = (logs.topic3)::text) OR (log_filters.topic3 IS NULL))); + + +-- +-- Name: blocks id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY blocks ALTER COLUMN id SET DEFAULT nextval('blocks_id_seq'::regclass); + + +-- +-- Name: log_filters id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY log_filters ALTER COLUMN id SET DEFAULT nextval('log_filters_id_seq'::regclass); + + +-- +-- Name: logs id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY logs ALTER COLUMN id SET DEFAULT nextval('logs_id_seq'::regclass); + + +-- +-- Name: nodes id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY nodes ALTER COLUMN id SET DEFAULT nextval('nodes_id_seq'::regclass); + + +-- +-- Name: receipts id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY receipts ALTER COLUMN id SET DEFAULT nextval('receipts_id_seq'::regclass); + + +-- +-- Name: transactions id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY transactions ALTER COLUMN id SET DEFAULT nextval('transactions_id_seq'::regclass); + + +-- +-- Name: watched_contracts contract_id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY watched_contracts ALTER COLUMN contract_id SET DEFAULT nextval('watched_contracts_contract_id_seq'::regclass); + + +-- +-- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY blocks + ADD CONSTRAINT blocks_pkey PRIMARY KEY (id); + + +-- +-- Name: watched_contracts contract_hash_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY watched_contracts + ADD CONSTRAINT contract_hash_uc UNIQUE (contract_hash); + + +-- +-- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY logs + ADD CONSTRAINT logs_pkey PRIMARY KEY (id); + + +-- +-- Name: log_filters name_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY log_filters + ADD CONSTRAINT name_uc UNIQUE (name); + + +-- +-- Name: blocks node_id_block_number_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY blocks + ADD CONSTRAINT node_id_block_number_uc UNIQUE (block_number, node_id); + + +-- +-- Name: nodes node_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY nodes + ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id); + + +-- +-- Name: nodes nodes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY nodes + ADD CONSTRAINT nodes_pkey PRIMARY KEY (id); + + +-- +-- Name: receipts receipts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY receipts + ADD CONSTRAINT receipts_pkey PRIMARY KEY (id); + + +-- +-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY schema_migrations + ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); + + +-- +-- Name: transactions transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY transactions + ADD CONSTRAINT transactions_pkey PRIMARY KEY (id); + + +-- +-- Name: watched_contracts watched_contracts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY watched_contracts + ADD CONSTRAINT watched_contracts_pkey PRIMARY KEY (contract_id); + + +-- +-- Name: block_id_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX block_id_index ON transactions USING btree (block_id); + + +-- +-- Name: block_number_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX block_number_index ON blocks USING btree (block_number); + + +-- +-- Name: node_id_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX node_id_index ON blocks USING btree (node_id); + + +-- +-- Name: transaction_id_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX transaction_id_index ON receipts USING btree (transaction_id); + + +-- +-- Name: tx_from_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX tx_from_index ON transactions USING btree (tx_from); + + +-- +-- Name: tx_to_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX tx_to_index ON transactions USING btree (tx_to); + + +-- +-- Name: transactions blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY transactions + ADD CONSTRAINT blocks_fk FOREIGN KEY (block_id) REFERENCES blocks(id) ON DELETE CASCADE; + + +-- +-- Name: blocks node_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY blocks + ADD CONSTRAINT node_fk FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE; + + +-- +-- Name: logs receipts_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY logs + ADD CONSTRAINT receipts_fk FOREIGN KEY (receipt_id) REFERENCES receipts(id) ON DELETE CASCADE; + + +-- +-- Name: receipts transaction_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY receipts + ADD CONSTRAINT transaction_fk FOREIGN KEY (transaction_id) REFERENCES transactions(id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + diff --git a/filters/example-filter.json b/filters/example-filter.json new file mode 100644 index 00000000..b123b0eb --- /dev/null +++ b/filters/example-filter.json @@ -0,0 +1,15 @@ +[{ +"name": "TransferFilter", +"fromBlock": "0x488290", +"toBlock": "0x488678", +"address": "0x06012c8cf97bead5deae237070f9587f8e7a266d", +"topics": ["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"] +}, +{ + "name": "NewFilter", + "toBlock": "0x4B34AA", + "fromBlock": "0x4B34AD", + "address": "0x06012c8cf97bead5deae237070f9587f8e7a266d", + "topics": ["0x241ea03ca20251805084d27d4440371c34a0b85ff108f6bb5611248f73818b80"] +}] + diff --git a/integration_test/contract_test.go b/integration_test/contract_test.go index 4aa65846..d6deb549 100644 --- a/integration_test/contract_test.go +++ b/integration_test/contract_test.go @@ -15,8 +15,7 @@ import ( var _ = Describe("Reading contracts", func() { - //TODO was experiencing Infura issue (I suspect) on 1/5. Unignore these and revisit if persists on next commit - XDescribe("Reading the list of attributes", func() { + Describe("Reading the list of attributes", func() { It("returns a string attribute for a real contract", func() { config, err := cfg.NewConfig("infura") if err != nil { @@ -59,8 +58,7 @@ var _ = Describe("Reading contracts", func() { }) }) - //TODO was experiencing Infura issue (I suspect) on 1/5. Unignore these and revisit if persists on next commit - XDescribe("Getting a contract attribute", func() { + Describe("Getting a contract attribute", func() { It("returns the correct attribute for a real contract", func() { config, _ := cfg.NewConfig("infura") blockchain := geth.NewBlockchain(config.Client.IPCPath) @@ -109,8 +107,8 @@ var _ = Describe("Reading contracts", func() { expectedLogZero := core.Log{ BlockNumber: 4703824, TxHash: "0xf896bfd1eb539d881a1a31102b78de9f25cd591bf1fe1924b86148c0b205fd5d", - Address: "0xd26114cd6EE289AccF82350c8d8487fedB8A0C07", - Topics: map[int]string{ + Address: "0xd26114cd6ee289accf82350c8d8487fedb8a0c07", + Topics: core.Topics{ 0: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", 1: "0x000000000000000000000000fbb1b73c4f0bda4f67dca266ce6ef42f520fbb98", 2: "0x000000000000000000000000d26114cd6ee289accf82350c8d8487fedb8a0c07", diff --git a/integration_test/geth_blockchain_test.go b/integration_test/geth_blockchain_test.go index 99631e7a..9331c606 100644 --- a/integration_test/geth_blockchain_test.go +++ b/integration_test/geth_blockchain_test.go @@ -52,6 +52,8 @@ var _ = Describe("Reading from the Geth blockchain", func() { Expect(node.GenesisBlock).To(Equal(devNetworkGenesisBlock)) Expect(node.NetworkId).To(Equal(devNetworkNodeId)) + Expect(len(node.Id)).To(Equal(128)) + Expect(node.ClientName).To(ContainSubstring("Geth")) close(done) }, 15) diff --git a/main.go b/main.go new file mode 100644 index 00000000..a27eb869 --- /dev/null +++ b/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/vulcanize/vulcanizedb/cmd" +) + +func main() { + cmd.Execute() +} diff --git a/pkg/core/log.go b/pkg/core/log.go index 9688b46c..984e0432 100644 --- a/pkg/core/log.go +++ b/pkg/core/log.go @@ -4,7 +4,7 @@ type Log struct { BlockNumber int64 TxHash string Address string - Topics map[int]string - Index int64 - Data string + Topics + Index int64 + Data string } diff --git a/pkg/core/node_info.go b/pkg/core/node_info.go index 79ae55fb..fe88c928 100644 --- a/pkg/core/node_info.go +++ b/pkg/core/node_info.go @@ -3,4 +3,6 @@ package core type Node struct { GenesisBlock string NetworkId float64 + Id string + ClientName string } diff --git a/pkg/core/topics.go b/pkg/core/topics.go new file mode 100644 index 00000000..2f1acb9d --- /dev/null +++ b/pkg/core/topics.go @@ -0,0 +1,3 @@ +package core + +type Topics [4]string diff --git a/pkg/core/transaction.go b/pkg/core/transaction.go index 0aa900ab..585b4935 100644 --- a/pkg/core/transaction.go +++ b/pkg/core/transaction.go @@ -9,5 +9,5 @@ type Transaction struct { GasLimit int64 GasPrice int64 Receipt - Value int64 + Value string } diff --git a/pkg/fakes/blockchain.go b/pkg/fakes/blockchain.go index 9c8738bb..97e8c0d6 100644 --- a/pkg/fakes/blockchain.go +++ b/pkg/fakes/blockchain.go @@ -50,7 +50,7 @@ func NewBlockchain() *Blockchain { blocks: make(map[int64]core.Block), logs: make(map[string][]core.Log), contractAttributes: make(map[string]map[string]string), - node: core.Node{GenesisBlock: "GENESIS"}, + node: core.Node{GenesisBlock: "GENESIS", NetworkId: 1, Id: "x123", ClientName: "Geth"}, } } diff --git a/pkg/filters/filter_query.go b/pkg/filters/filter_query.go new file mode 100644 index 00000000..7070edee --- /dev/null +++ b/pkg/filters/filter_query.go @@ -0,0 +1,64 @@ +package filters + +import ( + "encoding/json" + + "errors" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +type LogFilters []LogFilter + +type LogFilter struct { + Name string `json:"name"` + FromBlock int64 `json:"fromBlock"` + ToBlock int64 `json:"toBlock"` + Address string `json:"address"` + core.Topics `json:"topics"` +} + +func (filterQuery *LogFilter) UnmarshalJSON(input []byte) error { + type Alias LogFilter + + var err error + aux := &struct { + ToBlock string `json:"toBlock"` + FromBlock string `json:"fromBlock"` + *Alias + }{ + Alias: (*Alias)(filterQuery), + } + if err := json.Unmarshal(input, &aux); err != nil { + return err + } + if filterQuery.Name == "" { + return errors.New("filters: must provide name for logfilter") + } + filterQuery.ToBlock, err = filterQuery.unmarshalFromToBlock(aux.ToBlock) + if err != nil { + return errors.New("filters: invalid fromBlock") + } + filterQuery.FromBlock, err = filterQuery.unmarshalFromToBlock(aux.FromBlock) + if err != nil { + return errors.New("filters: invalid fromBlock") + } + if !common.IsHexAddress(filterQuery.Address) { + return errors.New("filters: invalid address") + } + + return nil +} + +func (filterQuery *LogFilter) unmarshalFromToBlock(auxBlock string) (int64, error) { + if auxBlock == "" { + return -1, nil + } + block, err := hexutil.DecodeUint64(auxBlock) + if err != nil { + return 0, errors.New("filters: invalid block arg") + } + return int64(block), nil +} diff --git a/pkg/filters/filter_test.go b/pkg/filters/filter_test.go new file mode 100644 index 00000000..3112afd6 --- /dev/null +++ b/pkg/filters/filter_test.go @@ -0,0 +1,125 @@ +package filters_test + +import ( + "encoding/json" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/filters" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Log filters", func() { + It("decodes web3 filter to LogFilter", func() { + + var logFilter filters.LogFilter + jsonFilter := []byte( + `{ + "name": "TestEvent", + "fromBlock": "0x1", + "toBlock": "0x488290", + "address": "0x8888f1f195afa192cfee860698584c030f4c9db1", + "topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null, "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null] + }`) + err := json.Unmarshal(jsonFilter, &logFilter) + + Expect(err).ToNot(HaveOccurred()) + Expect(logFilter.Name).To(Equal("TestEvent")) + Expect(logFilter.FromBlock).To(Equal(int64(1))) + Expect(logFilter.ToBlock).To(Equal(int64(4752016))) + Expect(logFilter.Address).To(Equal("0x8888f1f195afa192cfee860698584c030f4c9db1")) + Expect(logFilter.Topics).To(Equal( + core.Topics{ + "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "", + "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + ""})) + }) + + It("decodes array of web3 filters to []LogFilter", func() { + + logFilters := make([]filters.LogFilter, 0) + jsonFilter := []byte( + `[{ + "name": "TestEvent", + "fromBlock": "0x1", + "toBlock": "0x488290", + "address": "0x8888f1f195afa192cfee860698584c030f4c9db1", + "topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null, "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null] + }, + { + "name": "TestEvent2", + "fromBlock": "0x3", + "toBlock": "0x4", + "address": "0xd26114cd6EE289AccF82350c8d8487fedB8A0C07", + "topics": ["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", "0x0000000000000000000000006b0949d4c6edfe467db78241b7d5566f3c2bb43e", "0x0000000000000000000000005e44c3e467a49c9ca0296a9f130fc433041aaa28"] + }]`) + err := json.Unmarshal(jsonFilter, &logFilters) + + Expect(err).ToNot(HaveOccurred()) + Expect(len(logFilters)).To(Equal(2)) + Expect(logFilters[0].Name).To(Equal("TestEvent")) + Expect(logFilters[1].Name).To(Equal("TestEvent2")) + }) + + It("requires valid ethereum address", func() { + + var logFilter filters.LogFilter + jsonFilter := []byte( + `{ + "name": "TestEvent", + "fromBlock": "0x1", + "toBlock": "0x2", + "address": "0x8888f1f195afa192cf84c030f4c9db1", + "topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null, "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null] + }`) + err := json.Unmarshal(jsonFilter, &logFilter) + Expect(err).To(HaveOccurred()) + + }) + It("requires name", func() { + + var logFilter filters.LogFilter + jsonFilter := []byte( + `{ + "fromBlock": "0x1", + "toBlock": "0x2", + "address": "0x8888f1f195afa192cfee860698584c030f4c9db1", + "topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null, "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null] + }`) + err := json.Unmarshal(jsonFilter, &logFilter) + Expect(err).To(HaveOccurred()) + + }) + + It("maps missing fromBlock to -1", func() { + + var logFilter filters.LogFilter + jsonFilter := []byte( + `{ + "name": "TestEvent", + "toBlock": "0x2", + "address": "0x8888f1f195afa192cfee860698584c030f4c9db1", + "topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null, "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null] + }`) + err := json.Unmarshal(jsonFilter, &logFilter) + Expect(err).ToNot(HaveOccurred()) + Expect(logFilter.FromBlock).To(Equal(int64(-1))) + + }) + + It("maps missing toBlock to -1", func() { + var logFilter filters.LogFilter + jsonFilter := []byte( + `{ + "name": "TestEvent", + "address": "0x8888f1f195afa192cfee860698584c030f4c9db1", + "topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null, "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null] + }`) + err := json.Unmarshal(jsonFilter, &logFilter) + Expect(err).ToNot(HaveOccurred()) + Expect(logFilter.ToBlock).To(Equal(int64(-1))) + + }) + +}) diff --git a/pkg/filters/query_builder_suite_test.go b/pkg/filters/query_builder_suite_test.go new file mode 100644 index 00000000..42029812 --- /dev/null +++ b/pkg/filters/query_builder_suite_test.go @@ -0,0 +1,13 @@ +package filters_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestQueryBuilder(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "QueryBuilder Suite") +} diff --git a/pkg/geth/abi.go b/pkg/geth/abi.go index e84523fb..51b72103 100644 --- a/pkg/geth/abi.go +++ b/pkg/geth/abi.go @@ -38,6 +38,19 @@ func NewEtherScanClient(url string) *EtherScanApi { } +func GenUrl(network string) string { + switch network { + case "ropsten": + return "https://ropsten.etherscan.io" + case "kovan": + return "https://kovan.etherscan.io" + case "rinkeby": + return "https://rinkeby.etherscan.io" + default: + return "https://api.etherscan.io" + } +} + //https://api.etherscan.io/api?module=contract&action=getabi&address=%s func (e *EtherScanApi) GetAbi(contractHash string) (string, error) { target := new(Response) diff --git a/pkg/geth/abi_test.go b/pkg/geth/abi_test.go index 1098f640..f930661d 100644 --- a/pkg/geth/abi_test.go +++ b/pkg/geth/abi_test.go @@ -101,5 +101,22 @@ var _ = Describe("ABI files", func() { }) }) }) + + Describe("Generating etherscan endpoints based on network", func() { + It("should return the main endpoint as the default", func() { + url := geth.GenUrl("") + Expect(url).To(Equal("https://api.etherscan.io")) + }) + + It("generates various test network endpoint if test network is supplied", func() { + ropstenUrl := geth.GenUrl("ropsten") + rinkebyUrl := geth.GenUrl("rinkeby") + kovanUrl := geth.GenUrl("kovan") + + Expect(ropstenUrl).To(Equal("https://ropsten.etherscan.io")) + Expect(kovanUrl).To(Equal("https://kovan.etherscan.io")) + Expect(rinkebyUrl).To(Equal("https://rinkeby.etherscan.io")) + }) + }) }) }) diff --git a/pkg/geth/block_to_core_block.go b/pkg/geth/block_to_core_block.go index ee883bee..e09a8000 100644 --- a/pkg/geth/block_to_core_block.go +++ b/pkg/geth/block_to_core_block.go @@ -25,7 +25,7 @@ func ToCoreBlock(gethBlock *types.Block, client GethClient) core.Block { GasLimit: gethBlock.GasLimit().Int64(), GasUsed: gethBlock.GasUsed().Int64(), Hash: gethBlock.Hash().Hex(), - Miner: gethBlock.Coinbase().Hex(), + Miner: strings.ToLower(gethBlock.Coinbase().Hex()), Nonce: hexutil.Encode(gethBlock.Header().Nonce[:]), Number: gethBlock.Number().Int64(), ParentHash: gethBlock.ParentHash().Hex(), @@ -58,6 +58,10 @@ func convertTransactionsToCore(gethBlock *types.Block, client GethClient) []core func appendReceiptToTransaction(client GethClient, transaction core.Transaction) (core.Transaction, error) { gethReceipt, err := client.TransactionReceipt(context.Background(), common.HexToHash(transaction.Hash)) + if err != nil { + log.Println(err) + return transaction, err + } receipt := ReceiptToCoreReceipt(gethReceipt) transaction.Receipt = receipt return transaction, err @@ -72,7 +76,7 @@ func transToCoreTrans(transaction *types.Transaction, from *common.Address) core From: strings.ToLower(addressToHex(from)), GasLimit: transaction.Gas().Int64(), GasPrice: transaction.GasPrice().Int64(), - Value: transaction.Value().Int64(), + Value: transaction.Value().String(), Data: data, } } diff --git a/pkg/geth/block_to_core_block_test.go b/pkg/geth/block_to_core_block_test.go index 3cbe30e9..aae8e2c8 100644 --- a/pkg/geth/block_to_core_block_test.go +++ b/pkg/geth/block_to_core_block_test.go @@ -246,7 +246,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() { Expect(coreTransaction.From).To(Equal("0x0000000000000000000000000000000000000123")) Expect(coreTransaction.GasLimit).To(Equal(gethTransaction.Gas().Int64())) Expect(coreTransaction.GasPrice).To(Equal(gethTransaction.GasPrice().Int64())) - Expect(coreTransaction.Value).To(Equal(gethTransaction.Value().Int64())) + Expect(coreTransaction.Value).To(Equal(gethTransaction.Value().String())) Expect(coreTransaction.Nonce).To(Equal(gethTransaction.Nonce())) coreReceipt := coreTransaction.Receipt diff --git a/pkg/geth/blockchain.go b/pkg/geth/blockchain.go index 7c8aaa47..169698a9 100644 --- a/pkg/geth/blockchain.go +++ b/pkg/geth/blockchain.go @@ -3,6 +3,10 @@ package geth import ( "math/big" + "strings" + + "log" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/geth/node" "github.com/ethereum/go-ethereum" @@ -23,13 +27,27 @@ type Blockchain struct { func NewBlockchain(ipcPath string) *Blockchain { blockchain := Blockchain{} - rpcClient, _ := rpc.Dial(ipcPath) + rpcClient, err := rpc.Dial(ipcPath) + if err != nil { + log.Fatal(err) + } client := ethclient.NewClient(rpcClient) - blockchain.node = node.Retrieve(rpcClient) + blockchain.node = node.Info(rpcClient) + if infura := isInfuraNode(ipcPath); infura { + blockchain.node.Id = "infura" + blockchain.node.ClientName = "infura" + } blockchain.client = client return &blockchain } +func isInfuraNode(ipcPath string) bool { + if strings.Contains(ipcPath, "infura") { + return true + } + return false +} + func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]core.Log, error) { if endingBlockNumber == nil { endingBlockNumber = startingBlockNumber @@ -44,7 +62,7 @@ func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumbe if err != nil { return []core.Log{}, err } - logs := GethLogsToCoreLogs(gethLogs) + logs := ToCoreLogs(gethLogs) return logs, nil } diff --git a/pkg/geth/log_to_core_log.go b/pkg/geth/log_to_core_log.go index 2652898b..380ffc85 100644 --- a/pkg/geth/log_to_core_log.go +++ b/pkg/geth/log_to_core_log.go @@ -1,19 +1,36 @@ package geth import ( + "strings" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" ) -func LogToCoreLog(gethLog types.Log) core.Log { - topics := gethLog.Topics - var hexTopics = make(map[int]string) +func ToCoreLogs(gethLogs []types.Log) []core.Log { + var logs []core.Log + for _, log := range gethLogs { + log := ToCoreLog(log) + logs = append(logs, log) + } + return logs +} + +func makeTopics(topics []common.Hash) core.Topics { + var hexTopics core.Topics for i, topic := range topics { hexTopics[i] = topic.Hex() } + return hexTopics +} + +func ToCoreLog(gethLog types.Log) core.Log { + topics := gethLog.Topics + hexTopics := makeTopics(topics) return core.Log{ - Address: gethLog.Address.Hex(), + Address: strings.ToLower(gethLog.Address.Hex()), BlockNumber: int64(gethLog.BlockNumber), Topics: hexTopics, @@ -22,12 +39,3 @@ func LogToCoreLog(gethLog types.Log) core.Log { Data: hexutil.Encode(gethLog.Data), } } - -func GethLogsToCoreLogs(gethLogs []types.Log) []core.Log { - var logs []core.Log - for _, log := range gethLogs { - log := LogToCoreLog(log) - logs = append(logs, log) - } - return logs -} diff --git a/pkg/geth/log_to_core_log_test.go b/pkg/geth/log_to_core_log_test.go index daf91958..3601672c 100644 --- a/pkg/geth/log_to_core_log_test.go +++ b/pkg/geth/log_to_core_log_test.go @@ -1,6 +1,8 @@ package geth_test import ( + "strings" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/geth" "github.com/ethereum/go-ethereum/common" @@ -14,7 +16,7 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { It("converts geth log to internal log format", func() { gethLog := types.Log{ - Address: common.HexToAddress("0xecf8f87f810ecf450940c9f60066b4a7a501d6a7"), + Address: common.HexToAddress("0x448a5065aeBB8E423F0896E6c5D525C040f59af3"), BlockHash: common.HexToHash("0x656c34545f90a730a19008c0e7a7cd4fb3895064b48d6d69761bd5abad681056"), BlockNumber: 2019236, Data: hexutil.MustDecode("0x000000000000000000000000000000000000000000000001a055690d9db80000"), @@ -28,18 +30,18 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { } expected := core.Log{ - Address: gethLog.Address.Hex(), + Address: strings.ToLower(gethLog.Address.Hex()), BlockNumber: int64(gethLog.BlockNumber), Data: hexutil.Encode(gethLog.Data), TxHash: gethLog.TxHash.Hex(), Index: 2, - Topics: map[int]string{ - 0: common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef").Hex(), - 1: common.HexToHash("0x00000000000000000000000080b2c9d7cbbf30a1b0fc8983c647d754c6525615").Hex(), + Topics: core.Topics{ + gethLog.Topics[0].Hex(), + gethLog.Topics[1].Hex(), }, } - coreLog := geth.LogToCoreLog(gethLog) + coreLog := geth.ToCoreLog(gethLog) Expect(coreLog.Address).To(Equal(expected.Address)) Expect(coreLog.BlockNumber).To(Equal(expected.BlockNumber)) @@ -79,10 +81,10 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { }, } - expectedOne := geth.LogToCoreLog(gethLogOne) - expectedTwo := geth.LogToCoreLog(gethLogTwo) + expectedOne := geth.ToCoreLog(gethLogOne) + expectedTwo := geth.ToCoreLog(gethLogTwo) - coreLogs := geth.GethLogsToCoreLogs([]types.Log{gethLogOne, gethLogTwo}) + coreLogs := geth.ToCoreLogs([]types.Log{gethLogOne, gethLogTwo}) Expect(len(coreLogs)).To(Equal(2)) Expect(coreLogs[0]).To(Equal(expectedOne)) diff --git a/pkg/geth/node/node.go b/pkg/geth/node/node.go index 372cbfaa..c47e34bb 100644 --- a/pkg/geth/node/node.go +++ b/pkg/geth/node/node.go @@ -3,30 +3,47 @@ package node import ( "context" + "strconv" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rpc" ) -func Retrieve(client *rpc.Client) core.Node { - var info p2p.NodeInfo +func Info(client *rpc.Client) core.Node { node := core.Node{} - client.CallContext(context.Background(), &info, "admin_nodeInfo") - for protocolName, protocol := range info.Protocols { - if protocolName == "eth" { - protocolMap, _ := protocol.(map[string]interface{}) - node.GenesisBlock = getAttribute(protocolMap, "genesis").(string) - node.NetworkId = getAttribute(protocolMap, "network").(float64) - } - } + node.NetworkId = NetworkId(client) + node.GenesisBlock = GenesisBlock(client) + node.Id, node.ClientName = IdClientName(client) return node } -func getAttribute(protocolMap map[string]interface{}, protocol string) interface{} { - for key, val := range protocolMap { - if key == protocol { - return val - } +func IdClientName(client *rpc.Client) (string, string) { + var info p2p.NodeInfo + modules, _ := client.SupportedModules() + if _, ok := modules["admin"]; ok { + client.CallContext(context.Background(), &info, "admin_nodeInfo") + return info.ID, info.Name } - return nil + return "", "" +} + +func NetworkId(client *rpc.Client) float64 { + var version string + client.CallContext(context.Background(), &version, "net_version") + networkId, _ := strconv.ParseFloat(version, 64) + return networkId +} + +func ProtocolVersion(client *rpc.Client) string { + var protocolVersion string + client.CallContext(context.Background(), &protocolVersion, "eth_protocolVersion") + return protocolVersion +} + +func GenesisBlock(client *rpc.Client) string { + var header *types.Header + client.CallContext(context.Background(), &header, "eth_getBlockByNumber", "0x0", false) + return header.Hash().Hex() } diff --git a/pkg/geth/receipt_to_core_receipt.go b/pkg/geth/receipt_to_core_receipt.go index a597fd2e..b94ddb75 100644 --- a/pkg/geth/receipt_to_core_receipt.go +++ b/pkg/geth/receipt_to_core_receipt.go @@ -49,7 +49,7 @@ func setContractAddress(gethReceipt *types.Receipt) string { func dereferenceLogs(gethReceipt *types.Receipt) []core.Log { logs := []core.Log{} for _, log := range gethReceipt.Logs { - logs = append(logs, LogToCoreLog(*log)) + logs = append(logs, ToCoreLog(*log)) } return logs } diff --git a/pkg/geth/testing/sample_abi.json b/pkg/geth/testing/sample_abi.json index 3d80565a..a3582a41 100644 --- a/pkg/geth/testing/sample_abi.json +++ b/pkg/geth/testing/sample_abi.json @@ -1 +1 @@ -[{"constant":true,"inputs":[],"name":"mintingFinished","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"name","outputs":[{"name":"","type":"string"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_spender","type":"address"},{"name":"_value","type":"uint256"}],"name":"approve","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"totalSupply","outputs":[{"name":"","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_from","type":"address"},{"name":"_to","type":"address"},{"name":"_value","type":"uint256"}],"name":"transferFrom","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"decimals","outputs":[{"name":"","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"unpause","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_to","type":"address"},{"name":"_amount","type":"uint256"}],"name":"mint","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"paused","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_owner","type":"address"}],"name":"balanceOf","outputs":[{"name":"balance","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"finishMinting","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"pause","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"symbol","outputs":[{"name":"","type":"string"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_to","type":"address"},{"name":"_value","type":"uint256"}],"name":"transfer","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_to","type":"address"},{"name":"_amount","type":"uint256"},{"name":"_releaseTime","type":"uint256"}],"name":"mintTimelocked","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_owner","type":"address"},{"name":"_spender","type":"address"}],"name":"allowance","outputs":[{"name":"remaining","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"newOwner","type":"address"}],"name":"transferOwnership","outputs":[],"payable":false,"type":"function"},{"anonymous":false,"inputs":[{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Mint","type":"event"},{"anonymous":false,"inputs":[],"name":"MintFinished","type":"event"},{"anonymous":false,"inputs":[],"name":"Pause","type":"event"},{"anonymous":false,"inputs":[],"name":"Unpause","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"owner","type":"address"},{"indexed":true,"name":"spender","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Approval","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"from","type":"address"},{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Transfer","type":"event"}] +[{"constant":true,"inputs":[],"name":"mintingFinished","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"name","outputs":[{"name":"","type":"string"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_spender","type":"address"},{"name":"_value","type":"uint256"}],"name":"approve","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"totalSupply","outputs":[{"name":"","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_from","type":"address"},{"name":"_to","type":"address"},{"name":"_value","type":"uint256"}],"name":"transferFrom","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"decimals","outputs":[{"name":"","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"unpause","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_to","type":"address"},{"name":"_amount","type":"uint256"}],"name":"mint","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"paused","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_owner","type":"address"}],"name":"balanceOf","outputs":[{"name":"balance","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"finishMinting","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"pause","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"symbol","outputs":[{"name":"","type":"string"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_to","type":"address"},{"name":"_value","type":"uint256"}],"name":"transfer","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_to","type":"address"},{"name":"_amount","type":"uint256"},{"name":"_releaseTime","type":"uint256"}],"name":"mintTimelocked","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_owner","type":"address"},{"name":"_spender","type":"address"}],"name":"allowance","outputs":[{"name":"remaining","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"newOwner","type":"address"}],"name":"transferOwnership","outputs":[],"payable":false,"type":"function"},{"anonymous":false,"inputs":[{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Mint","type":"event"},{"anonymous":false,"inputs":[],"name":"MintFinished","type":"event"},{"anonymous":false,"inputs":[],"name":"Pause","type":"event"},{"anonymous":false,"inputs":[],"name":"Unpause","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"owner","type":"address"},{"indexed":true,"name":"spender","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Approval","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"from","type":"address"},{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Transfer","type":"event"}] \ No newline at end of file diff --git a/pkg/repositories/in_memory.go b/pkg/repositories/in_memory.go index 1148e1c4..2b1d0014 100644 --- a/pkg/repositories/in_memory.go +++ b/pkg/repositories/in_memory.go @@ -3,7 +3,10 @@ package repositories import ( "fmt" + "errors" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/filters" ) type InMemory struct { @@ -11,16 +14,27 @@ type InMemory struct { receipts map[string]core.Receipt contracts map[string]core.Contract logs map[string][]core.Log + logFilters map[string]filters.LogFilter CreateOrUpdateBlockCallCount int } +func (repository *InMemory) AddFilter(filter filters.LogFilter) error { + key := filter.Name + if _, ok := repository.logFilters[key]; ok || key == "" { + return errors.New("filter name not unique") + } + repository.logFilters[key] = filter + return nil +} + func NewInMemory() *InMemory { return &InMemory{ CreateOrUpdateBlockCallCount: 0, - blocks: make(map[int64]core.Block), - receipts: make(map[string]core.Receipt), - contracts: make(map[string]core.Contract), - logs: make(map[string][]core.Log), + blocks: make(map[int64]core.Block), + receipts: make(map[string]core.Receipt), + contracts: make(map[string]core.Contract), + logs: make(map[string][]core.Log), + logFilters: make(map[string]filters.LogFilter), } } @@ -102,6 +116,7 @@ func (repository *InMemory) CreateOrUpdateBlock(block core.Block) error { repository.blocks[block.Number] = block for _, transaction := range block.Transactions { repository.receipts[transaction.Hash] = transaction.Receipt + repository.logs[transaction.TxHash] = transaction.Logs } return nil } diff --git a/pkg/repositories/postgres.go b/pkg/repositories/postgres.go index 5166b003..4892f88b 100644 --- a/pkg/repositories/postgres.go +++ b/pkg/repositories/postgres.go @@ -11,6 +11,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/filters" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" ) @@ -62,35 +63,6 @@ func (repository Postgres) SetBlocksStatus(chainHead int64) { cutoff) } -func (repository Postgres) CreateLogs(logs []core.Log) error { - tx, _ := repository.Db.BeginTx(context.Background(), nil) - for _, tlog := range logs { - _, err := tx.Exec( - `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ON CONFLICT (index, block_number) - DO UPDATE - SET block_number = $1, - address = $2, - tx_hash = $3, - index = $4, - topic0 = $5, - topic1 = $6, - topic2 = $7, - topic3 = $8, - data = $9 - `, - tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, - ) - if err != nil { - tx.Rollback() - return ErrDBInsertFailed - } - } - tx.Commit() - return nil -} - func (repository Postgres) FindLogs(address string, blockNumber int64) []core.Log { logRows, _ := repository.Db.Query( `SELECT block_number, @@ -111,13 +83,16 @@ func (repository Postgres) FindLogs(address string, blockNumber int64) []core.Lo func (repository *Postgres) CreateNode(node *core.Node) error { var nodeId int64 err := repository.Db.QueryRow( - `INSERT INTO nodes (genesis_block, network_id) - VALUES ($1, $2) - ON CONFLICT (genesis_block, network_id) + `INSERT INTO nodes (genesis_block, network_id, node_id, client_name) + VALUES ($1, $2, $3, $4) + ON CONFLICT (genesis_block, network_id, node_id) DO UPDATE - SET genesis_block = $1, network_id = $2 + SET genesis_block = $1, + network_id = $2, + node_id = $3, + client_name = $4 RETURNING id`, - node.GenesisBlock, node.NetworkId).Scan(&nodeId) + node.GenesisBlock, node.NetworkId, node.Id, node.ClientName).Scan(&nodeId) if err != nil { return ErrUnableToSetNode } @@ -329,29 +304,90 @@ func (repository Postgres) createTransaction(tx *sql.Tx, blockId int64, transact err := tx.QueryRow( `INSERT INTO transactions (block_id, tx_hash, tx_nonce, tx_to, tx_from, tx_gaslimit, tx_gasprice, tx_value, tx_input_data) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + VALUES ($1, $2, $3, $4, $5, $6, $7, cast(NULLIF($8, '') AS NUMERIC), $9) RETURNING id`, blockId, transaction.Hash, transaction.Nonce, transaction.To, transaction.From, transaction.GasLimit, transaction.GasPrice, transaction.Value, transaction.Data). Scan(&transactionId) if err != nil { return err } - if transaction.Receipt.TxHash != "" { - err = repository.createReceipt(tx, transactionId, transaction.Receipt) + if hasReceipt(transaction) { + receiptId, err := repository.createReceipt(tx, transactionId, transaction.Receipt) if err != nil { return err } + if hasLogs(transaction) { + err = repository.createLogs(tx, transaction.Receipt.Logs, receiptId) + if err != nil { + return err + } + } } return nil } -func (repository Postgres) createReceipt(tx *sql.Tx, transactionId int, receipt core.Receipt) error { +func hasLogs(transaction core.Transaction) bool { + return len(transaction.Receipt.Logs) > 0 +} + +func hasReceipt(transaction core.Transaction) bool { + return transaction.Receipt.TxHash != "" +} + +func (repository Postgres) createReceipt(tx *sql.Tx, transactionId int, receipt core.Receipt) (int, error) { //Not currently persisting log bloom filters - _, err := tx.Exec( + var receiptId int + err := tx.QueryRow( `INSERT INTO receipts (contract_address, tx_hash, cumulative_gas_used, gas_used, state_root, status, transaction_id) - VALUES ($1, $2, $3, $4, $5, $6, $7)`, - receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, transactionId) + VALUES ($1, $2, $3, $4, $5, $6, $7) + RETURNING id`, + receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, transactionId).Scan(&receiptId) + if err != nil { + return receiptId, err + } + return receiptId, nil +} + +func (repository Postgres) createLogs(tx *sql.Tx, logs []core.Log, receiptId int) error { + for _, tlog := range logs { + _, err := tx.Exec( + `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + `, + tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, receiptId, + ) + if err != nil { + return ErrDBInsertFailed + } + } + return nil +} + +func (repository Postgres) CreateLogs(logs []core.Log) error { + tx, _ := repository.Db.BeginTx(context.Background(), nil) + for _, tlog := range logs { + _, err := tx.Exec( + `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + `, + tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, + ) + if err != nil { + tx.Rollback() + return ErrDBInsertFailed + } + } + tx.Commit() + return nil +} + +func (repository Postgres) AddFilter(query filters.LogFilter) error { + _, err := repository.Db.Exec( + `INSERT INTO log_filters + (name, from_block, to_block, address, topic0, topic1, topic2, topic3) + VALUES ($1, NULLIF($2, -1), NULLIF($3, -1), $4, NULLIF($5, ''), NULLIF($6, ''), NULLIF($7, ''), NULLIF($8, ''))`, + query.Name, query.FromBlock, query.ToBlock, query.Address, query.Topics[0], query.Topics[1], query.Topics[2], query.Topics[3]) if err != nil { return err } @@ -439,7 +475,7 @@ func (repository Postgres) loadLogs(logsRows *sql.Rows) []core.Log { var txHash string var index int64 var data string - topics := make([]string, 4) + var topics core.Topics logsRows.Scan(&blockNumber, &address, &txHash, &index, &topics[0], &topics[1], &topics[2], &topics[3], &data) log := core.Log{ BlockNumber: blockNumber, @@ -448,7 +484,6 @@ func (repository Postgres) loadLogs(logsRows *sql.Rows) []core.Log { Index: index, Data: data, } - log.Topics = make(map[int]string) for i, topic := range topics { log.Topics[i] = topic } @@ -467,7 +502,7 @@ func (repository Postgres) loadTransactions(transactionRows *sql.Rows) []core.Tr var gasLimit int64 var gasPrice int64 var inputData string - var value int64 + var value string transactionRows.Scan(&hash, &nonce, &to, &from, &gasLimit, &gasPrice, &value, &inputData) transaction := core.Transaction{ Hash: hash, diff --git a/pkg/repositories/postgres_test.go b/pkg/repositories/postgres_test.go index 04f3121c..9315b0b4 100644 --- a/pkg/repositories/postgres_test.go +++ b/pkg/repositories/postgres_test.go @@ -7,6 +7,8 @@ import ( "io/ioutil" "log" + "math/big" + "github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/repositories" @@ -38,6 +40,42 @@ var _ = Describe("Postgres repository", func() { return repository }) + It("serializes big.Int to db", func() { + // postgres driver doesn't support go big.Int type + // various casts in golang uint64, int64, overflow for + // transaction value (in wei) even though + // postgres numeric can handle an arbitrary + // sized int, so use string representation of big.Int + // and cast on insert + + cfg, _ := config.NewConfig("private") + pgConfig := config.DbConnectionString(cfg.Database) + db, err := sqlx.Connect("postgres", pgConfig) + + bi := new(big.Int) + bi.SetString("34940183920000000000", 10) + Expect(bi.String()).To(Equal("34940183920000000000")) + + defer db.Exec(`DROP TABLE IF EXISTS example`) + _, err = db.Exec("CREATE TABLE example ( id INTEGER, data NUMERIC )") + Expect(err).ToNot(HaveOccurred()) + + sqlStatement := ` + INSERT INTO example (id, data) + VALUES (1, cast($1 AS NUMERIC))` + _, err = db.Exec(sqlStatement, bi.String()) + Expect(err).ToNot(HaveOccurred()) + + var data string + err = db.QueryRow(`SELECT data FROM example WHERE id = 1`).Scan(&data) + Expect(err).ToNot(HaveOccurred()) + + Expect(bi.String()).To(Equal(data)) + actual := new(big.Int) + actual.SetString(data, 10) + Expect(actual).To(Equal(bi)) + }) + It("does not commit block if block is invalid", func() { //badNonce violates db Nonce field length badNonce := fmt.Sprintf("x %s", strings.Repeat("1", 100)) @@ -47,7 +85,7 @@ var _ = Describe("Postgres repository", func() { Transactions: []core.Transaction{}, } cfg, _ := config.NewConfig("private") - node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1} + node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1, Id: "x123", ClientName: "geth"} repository, _ := repositories.NewPostgres(cfg.Database, node) err1 := repository.CreateOrUpdateBlock(badBlock) @@ -60,7 +98,7 @@ var _ = Describe("Postgres repository", func() { It("throws error when can't connect to the database", func() { invalidDatabase := config.Database{} - node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1} + node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1, Id: "x123", ClientName: "geth"} _, err := repositories.NewPostgres(invalidDatabase, node) Expect(err).To(Equal(repositories.ErrDBConnectionFailed)) }) @@ -68,7 +106,7 @@ var _ = Describe("Postgres repository", func() { It("throws error when can't create node", func() { cfg, _ := config.NewConfig("private") badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100)) - node := core.Node{GenesisBlock: badHash, NetworkId: 1} + node := core.Node{GenesisBlock: badHash, NetworkId: 1, Id: "x123", ClientName: "geth"} _, err := repositories.NewPostgres(cfg.Database, node) Expect(err).To(Equal(repositories.ErrUnableToSetNode)) }) @@ -82,7 +120,7 @@ var _ = Describe("Postgres repository", func() { TxHash: badTxHash, } cfg, _ := config.NewConfig("private") - node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1} + node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1, Id: "x123", ClientName: "geth"} repository, _ := repositories.NewPostgres(cfg.Database, node) err := repository.CreateLogs([]core.Log{badLog}) @@ -101,7 +139,7 @@ var _ = Describe("Postgres repository", func() { Transactions: []core.Transaction{badTransaction}, } cfg, _ := config.NewConfig("private") - node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1} + node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1, Id: "x123", ClientName: "geth"} repository, _ := repositories.NewPostgres(cfg.Database, node) err1 := repository.CreateOrUpdateBlock(block) diff --git a/pkg/repositories/repository.go b/pkg/repositories/repository.go index 74f1f24e..83912f96 100644 --- a/pkg/repositories/repository.go +++ b/pkg/repositories/repository.go @@ -1,6 +1,9 @@ package repositories -import "github.com/vulcanize/vulcanizedb/pkg/core" +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/filters" +) const ( blocksFromHeadBeforeFinal = 20 @@ -19,4 +22,5 @@ type Repository interface { CreateLogs(log []core.Log) error FindLogs(address string, blockNumber int64) []core.Log SetBlocksStatus(chainHead int64) + AddFilter(filter filters.LogFilter) error } diff --git a/pkg/repositories/testing/helpers.go b/pkg/repositories/testing/helpers.go index 6ebc35aa..37937872 100644 --- a/pkg/repositories/testing/helpers.go +++ b/pkg/repositories/testing/helpers.go @@ -4,7 +4,10 @@ import ( "sort" "strconv" + "math/big" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/filters" "github.com/vulcanize/vulcanizedb/pkg/repositories" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -16,13 +19,19 @@ func ClearData(postgres repositories.Postgres) { postgres.Db.MustExec("DELETE FROM blocks") postgres.Db.MustExec("DELETE FROM logs") postgres.Db.MustExec("DELETE FROM receipts") + postgres.Db.MustExec("DELETE FROM log_filters") } func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories.Repository) { var repository repositories.Repository BeforeEach(func() { - node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1} + node := core.Node{ + GenesisBlock: "GENESIS", + NetworkId: 1, + Id: "b6f90c0fdd8ec9607aed8ee45c69322e47b7063f0bfb7a29c8ecafab24d0a22d24dd2329b5ee6ed4125a03cb14e57fd584e67f9e53e6c631055cbbd82f080845", + ClientName: "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9", + } repository = buildRepository(node) }) @@ -48,6 +57,8 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. nodeTwo := core.Node{ GenesisBlock: "0x456", NetworkId: 1, + Id: "x123456", + ClientName: "Geth", } repositoryTwo := buildRepository(nodeTwo) @@ -191,7 +202,8 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. nonce := uint64(10000) to := "1234567890" from := "0987654321" - value := int64(10) + var value = new(big.Int) + value.SetString("34940183920000000000", 10) inputData := "0xf7d8c8830000000000000000000000000000000000000000000000000000000000037788000000000000000000000000000000000000000000000000000000000003bd14" transaction := core.Transaction{ Hash: "x1234", @@ -200,7 +212,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Nonce: nonce, To: to, From: from, - Value: value, + Value: value.String(), Data: inputData, } block := core.Block{ @@ -220,7 +232,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Expect(savedTransaction.Nonce).To(Equal(nonce)) Expect(savedTransaction.GasLimit).To(Equal(gasLimit)) Expect(savedTransaction.GasPrice).To(Equal(gasPrice)) - Expect(savedTransaction.Value).To(Equal(value)) + Expect(savedTransaction.Value).To(Equal(value.String())) }) }) @@ -392,7 +404,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, Data: "xabc", }}, ) @@ -415,37 +427,13 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Expect(log).To(BeNil()) }) - It("updates the log when log with when log with same block number and index is already present", func() { - repository.CreateLogs([]core.Log{{ - BlockNumber: 1, - Index: 0, - Address: "x123", - TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, - Data: "xABC", - }, - }) - repository.CreateLogs([]core.Log{{ - BlockNumber: 1, - Index: 0, - Address: "x123", - TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, - Data: "xXYZ", - }, - }) - - log := repository.FindLogs("x123", 1) - Expect(log[0].Data).To(Equal("xXYZ")) - }) - It("filters to the correct block number and address", func() { repository.CreateLogs([]core.Log{{ BlockNumber: 1, Index: 0, Address: "x123", TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, Data: "xabc", }}, ) @@ -454,7 +442,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 1, Address: "x123", TxHash: "x789", - Topics: map[int]string{0: "x111", 1: "x222", 2: "x333"}, + Topics: core.Topics{0: "x111", 1: "x222", 2: "x333"}, Data: "xdef", }}, ) @@ -463,7 +451,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, Data: "xabc", }}, ) @@ -497,6 +485,85 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. {blockNumber: 1, Index: 1}}, )) }) + + It("saves the logs attached to a receipt", func() { + logs := []core.Log{{ + Address: "0x8a4774fe82c63484afef97ca8d89a6ea5e21f973", + BlockNumber: 4745407, + Data: "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000645a68669900000000000000000000000000000000000000000000003397684ab5869b0000000000000000000000000000000000000000000000000000000000005a36053200000000000000000000000099041f808d598b782d5a3e498681c2452a31da08", + Index: 86, + Topics: core.Topics{ + 0: "0x5a68669900000000000000000000000000000000000000000000000000000000", + 1: "0x000000000000000000000000d0148dad63f73ce6f1b6c607e3413dcf1ff5f030", + 2: "0x00000000000000000000000000000000000000000000003397684ab5869b0000", + 3: "0x000000000000000000000000000000000000000000000000000000005a360532", + }, + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + }, { + Address: "0x99041f808d598b782d5a3e498681c2452a31da08", + BlockNumber: 4745407, + Data: "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000418178358", + Index: 87, + Topics: core.Topics{ + 0: "0x1817835800000000000000000000000000000000000000000000000000000000", + 1: "0x0000000000000000000000008a4774fe82c63484afef97ca8d89a6ea5e21f973", + 2: "0x0000000000000000000000000000000000000000000000000000000000000000", + 3: "0x0000000000000000000000000000000000000000000000000000000000000000", + }, + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + }, { + Address: "0x99041f808d598b782d5a3e498681c2452a31da08", + BlockNumber: 4745407, + Data: "0x00000000000000000000000000000000000000000000003338f64c8423af4000", + Index: 88, + Topics: core.Topics{ + 0: "0x296ba4ca62c6c21c95e828080cb8aec7481b71390585605300a8a76f9e95b527", + }, + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + }, + } + receipt := core.Receipt{ + ContractAddress: "", + CumulativeGasUsed: 7481414, + GasUsed: 60711, + Logs: logs, + Bloom: "0x00000800000000000000001000000000000000400000000080000000000000000000400000010000000000000000000000000000040000000000000002000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000800004000000000000001000000000000000000000000000002000000480000000000000002000000000000000020000000000000000000000000000000000000000080000000000180000c00000000000000002000002000000040000000000000000000000000000010000000000020000000000000000000002000000000000000000000000400800000000000000000", + Status: 1, + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + } + transaction := + core.Transaction{ + Hash: receipt.TxHash, + Receipt: receipt, + } + + block := core.Block{Transactions: []core.Transaction{transaction}} + err := repository.CreateOrUpdateBlock(block) + Expect(err).To(Not(HaveOccurred())) + retrievedLogs := repository.FindLogs("0x99041f808d598b782d5a3e498681c2452a31da08", 4745407) + + expected := logs[1:] + Expect(retrievedLogs).To(Equal(expected)) + }) + + It("still saves receipts without logs", func() { + receipt := core.Receipt{ + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + } + transaction := core.Transaction{ + Hash: receipt.TxHash, + Receipt: receipt, + } + + block := core.Block{ + Transactions: []core.Transaction{transaction}, + } + repository.CreateOrUpdateBlock(block) + + _, err := repository.FindReceipt(receipt.TxHash) + + Expect(err).To(Not(HaveOccurred())) + }) }) Describe("Saving receipts", func() { @@ -535,6 +602,43 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Expect(err).To(HaveOccurred()) Expect(receipt).To(BeZero()) }) + }) + Describe("LogFilter", func() { + + It("inserts filter into watched events", func() { + + logFilter := filters.LogFilter{ + Name: "TestFilter", + FromBlock: 1, + ToBlock: 2, + Address: "0x8888f1f195afa192cfee860698584c030f4c9db1", + Topics: core.Topics{ + "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "", + "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "", + }, + } + err := repository.AddFilter(logFilter) + Expect(err).ToNot(HaveOccurred()) + }) + + It("returns error if name is not provided", func() { + + logFilter := filters.LogFilter{ + FromBlock: 1, + ToBlock: 2, + Address: "0x8888f1f195afa192cfee860698584c030f4c9db1", + Topics: core.Topics{ + "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "", + "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "", + }, + } + err := repository.AddFilter(logFilter) + Expect(err).To(HaveOccurred()) + }) }) } diff --git a/cmd/utils.go b/utils/utils.go similarity index 75% rename from cmd/utils.go rename to utils/utils.go index 8c1543a7..3aa73579 100644 --- a/cmd/utils.go +++ b/utils/utils.go @@ -1,14 +1,14 @@ -package cmd +package utils import ( "log" "path/filepath" - "fmt" - "math/big" + "os" + "github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/geth" @@ -32,9 +32,7 @@ func LoadPostgres(database config.Database, node core.Node) repositories.Postgre } func ReadAbiFile(abiFilepath string) string { - if !filepath.IsAbs(abiFilepath) { - abiFilepath = filepath.Join(config.ProjectRoot(), abiFilepath) - } + abiFilepath = AbsFilePath(abiFilepath) abi, err := geth.ReadAbiFile(abiFilepath) if err != nil { log.Fatalf("Error reading ABI file at \"%s\"\n %v", abiFilepath, err) @@ -42,13 +40,22 @@ func ReadAbiFile(abiFilepath string) string { return abi } -func GetAbi(abiFilepath string, contractHash string) string { +func AbsFilePath(filePath string) string { + if !filepath.IsAbs(filePath) { + cwd, _ := os.Getwd() + filePath = filepath.Join(cwd, filePath) + } + return filePath +} + +func GetAbi(abiFilepath string, contractHash string, network string) string { var contractAbiString string if abiFilepath != "" { contractAbiString = ReadAbiFile(abiFilepath) } else { - etherscan := geth.NewEtherScanClient("https://api.etherscan.io") - fmt.Println("No ABI supplied. Retrieving ABI from Etherscan") + url := geth.GenUrl(network) + etherscan := geth.NewEtherScanClient(url) + log.Printf("No ABI supplied. Retrieving ABI from Etherscan: %s", url) contractAbiString, _ = etherscan.GetAbi(contractHash) } _, err := geth.ParseAbi(contractAbiString)