From 54c4f0c2fedc06bf41770c8356771dd8ab6683b0 Mon Sep 17 00:00:00 2001 From: Matt K <1036969+mkrump@users.noreply.github.com> Date: Mon, 8 Jan 2018 11:41:01 -0600 Subject: [PATCH 01/22] Add back infura related tests (#123) --- integration_test/contract_test.go | 6 ++---- pkg/geth/testing/sample_abi.json | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/integration_test/contract_test.go b/integration_test/contract_test.go index e0155f64..6d0af83f 100644 --- a/integration_test/contract_test.go +++ b/integration_test/contract_test.go @@ -15,8 +15,7 @@ import ( var _ = Describe("Reading contracts", func() { - //TODO was experiencing Infura issue (I suspect) on 1/5. Unignore these and revisit if persists on next commit - XDescribe("Reading the list of attributes", func() { + Describe("Reading the list of attributes", func() { It("returns a string attribute for a real contract", func() { config, err := cfg.NewConfig("infura") if err != nil { @@ -59,8 +58,7 @@ var _ = Describe("Reading contracts", func() { }) }) - //TODO was experiencing Infura issue (I suspect) on 1/5. Unignore these and revisit if persists on next commit - XDescribe("Getting a contract attribute", func() { + Describe("Getting a contract attribute", func() { It("returns the correct attribute for a real contract", func() { config, _ := cfg.NewConfig("infura") blockchain := geth.NewBlockchain(config.Client.IPCPath) diff --git a/pkg/geth/testing/sample_abi.json b/pkg/geth/testing/sample_abi.json index 3d80565a..a3582a41 100644 --- a/pkg/geth/testing/sample_abi.json +++ b/pkg/geth/testing/sample_abi.json @@ -1 +1 @@ -[{"constant":true,"inputs":[],"name":"mintingFinished","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"name","outputs":[{"name":"","type":"string"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_spender","type":"address"},{"name":"_value","type":"uint256"}],"name":"approve","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"totalSupply","outputs":[{"name":"","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_from","type":"address"},{"name":"_to","type":"address"},{"name":"_value","type":"uint256"}],"name":"transferFrom","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"decimals","outputs":[{"name":"","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"unpause","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_to","type":"address"},{"name":"_amount","type":"uint256"}],"name":"mint","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"paused","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_owner","type":"address"}],"name":"balanceOf","outputs":[{"name":"balance","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"finishMinting","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"pause","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"symbol","outputs":[{"name":"","type":"string"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_to","type":"address"},{"name":"_value","type":"uint256"}],"name":"transfer","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_to","type":"address"},{"name":"_amount","type":"uint256"},{"name":"_releaseTime","type":"uint256"}],"name":"mintTimelocked","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_owner","type":"address"},{"name":"_spender","type":"address"}],"name":"allowance","outputs":[{"name":"remaining","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"newOwner","type":"address"}],"name":"transferOwnership","outputs":[],"payable":false,"type":"function"},{"anonymous":false,"inputs":[{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Mint","type":"event"},{"anonymous":false,"inputs":[],"name":"MintFinished","type":"event"},{"anonymous":false,"inputs":[],"name":"Pause","type":"event"},{"anonymous":false,"inputs":[],"name":"Unpause","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"owner","type":"address"},{"indexed":true,"name":"spender","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Approval","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"from","type":"address"},{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Transfer","type":"event"}] +[{"constant":true,"inputs":[],"name":"mintingFinished","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"name","outputs":[{"name":"","type":"string"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_spender","type":"address"},{"name":"_value","type":"uint256"}],"name":"approve","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"totalSupply","outputs":[{"name":"","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_from","type":"address"},{"name":"_to","type":"address"},{"name":"_value","type":"uint256"}],"name":"transferFrom","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"decimals","outputs":[{"name":"","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"unpause","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_to","type":"address"},{"name":"_amount","type":"uint256"}],"name":"mint","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"paused","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_owner","type":"address"}],"name":"balanceOf","outputs":[{"name":"balance","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"finishMinting","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"pause","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"symbol","outputs":[{"name":"","type":"string"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_to","type":"address"},{"name":"_value","type":"uint256"}],"name":"transfer","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_to","type":"address"},{"name":"_amount","type":"uint256"},{"name":"_releaseTime","type":"uint256"}],"name":"mintTimelocked","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_owner","type":"address"},{"name":"_spender","type":"address"}],"name":"allowance","outputs":[{"name":"remaining","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"newOwner","type":"address"}],"name":"transferOwnership","outputs":[],"payable":false,"type":"function"},{"anonymous":false,"inputs":[{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Mint","type":"event"},{"anonymous":false,"inputs":[],"name":"MintFinished","type":"event"},{"anonymous":false,"inputs":[],"name":"Pause","type":"event"},{"anonymous":false,"inputs":[],"name":"Unpause","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"owner","type":"address"},{"indexed":true,"name":"spender","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Approval","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"from","type":"address"},{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Transfer","type":"event"}] \ No newline at end of file From 14e1fc4213605ff31ccc706a606e24710ec49748 Mon Sep 17 00:00:00 2001 From: Matt K <1036969+mkrump@users.noreply.github.com> Date: Mon, 8 Jan 2018 14:19:42 -0600 Subject: [PATCH 02/22] Remove admin api dependency (#126) --- pkg/geth/block_to_core_block.go | 4 ++++ pkg/geth/node/node.go | 34 +++++++++++++++------------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/pkg/geth/block_to_core_block.go b/pkg/geth/block_to_core_block.go index c82000dd..34df1660 100644 --- a/pkg/geth/block_to_core_block.go +++ b/pkg/geth/block_to_core_block.go @@ -58,6 +58,10 @@ func convertTransactionsToCore(gethBlock *types.Block, client GethClient) []core func appendReceiptToTransaction(client GethClient, transaction core.Transaction) (core.Transaction, error) { gethReceipt, err := client.TransactionReceipt(context.Background(), common.HexToHash(transaction.Hash)) + if err != nil { + log.Println(err) + return transaction, err + } receipt := ReceiptToCoreReceipt(gethReceipt) transaction.Receipt = receipt return transaction, err diff --git a/pkg/geth/node/node.go b/pkg/geth/node/node.go index 38fcfe6d..524316dd 100644 --- a/pkg/geth/node/node.go +++ b/pkg/geth/node/node.go @@ -3,30 +3,26 @@ package node import ( "context" + "strconv" + "github.com/8thlight/vulcanizedb/pkg/core" - "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" ) func Retrieve(client *rpc.Client) core.Node { - var info p2p.NodeInfo node := core.Node{} - client.CallContext(context.Background(), &info, "admin_nodeInfo") - for protocolName, protocol := range info.Protocols { - if protocolName == "eth" { - protocolMap, _ := protocol.(map[string]interface{}) - node.GenesisBlock = getAttribute(protocolMap, "genesis").(string) - node.NetworkId = getAttribute(protocolMap, "network").(float64) - } - } + + var version string + client.CallContext(context.Background(), &version, "net_version") + node.NetworkId, _ = strconv.ParseFloat(version, 64) + + var protocolVersion string + client.CallContext(context.Background(), &protocolVersion, "eth_protocolVersion") + + var header *types.Header + client.CallContext(context.Background(), &header, "eth_getBlockByNumber", "0x0", false) + node.GenesisBlock = header.Hash().Hex() + return node } - -func getAttribute(protocolMap map[string]interface{}, protocol string) interface{} { - for key, val := range protocolMap { - if key == protocol { - return val - } - } - return nil -} From 70cfa20c682f578679c8ea3122988f98916e0d4b Mon Sep 17 00:00:00 2001 From: Matt K <1036969+mkrump@users.noreply.github.com> Date: Mon, 8 Jan 2018 15:59:47 -0600 Subject: [PATCH 03/22] Watch contact updates (#127) * Downcase all arguments for contact watching * ABI retrieval from test networks --- Gododir/main.go | 4 +++- README.md | 4 ++-- cmd/get_logs/main.go | 11 +++++++---- cmd/run/main.go | 5 +++-- cmd/show_contract_summary/main.go | 6 +++++- cmd/utils.go | 9 ++++----- cmd/vulcanize_db/main.go | 5 +++-- cmd/watch_contract/main.go | 11 ++++++++--- pkg/geth/abi.go | 13 +++++++++++++ pkg/geth/abi_test.go | 17 +++++++++++++++++ pkg/geth/block_to_core_block.go | 2 +- 11 files changed, 66 insertions(+), 21 deletions(-) diff --git a/Gododir/main.go b/Gododir/main.go index 6acd911b..09b49130 100644 --- a/Gododir/main.go +++ b/Gododir/main.go @@ -60,14 +60,16 @@ func tasks(p *do.Project) { environment := parseEnvironment(context) contractHash := context.Args.MayString("", "contract-hash", "c") abiFilepath := context.Args.MayString("", "abi-filepath", "a") + network := context.Args.MayString("", "network", "n") if contractHash == "" { log.Fatalln("--contract-hash required") } - context.Start(`go run main.go --environment={{.environment}} --contract-hash={{.contractHash}} --abi-filepath={{.abiFilepath}}`, + context.Start(`go run main.go --environment={{.environment}} --contract-hash={{.contractHash}} --abi-filepath={{.abiFilepath}} --network={{.network}}`, do.M{ "environment": environment, "contractHash": contractHash, "abiFilepath": abiFilepath, + "network": network, "$in": "cmd/watch_contract", }) }) diff --git a/README.md b/README.md index 5af21eaa..9ff7ea88 100644 --- a/README.md +++ b/README.md @@ -86,8 +86,8 @@ vulcanizedb/ .json private/ ``` -The name of the JSON file should correspond the contract's address. -2. Start watching the contract `godo watchContract -- --environment= --contract-hash=` +The name of the JSON file should correspond the contract's address. +2. Start watching the contract `godo watchContract -- --environment= --contract-hash=` 3. Request summary data `godo showContractSummary -- --environment= --contract-hash=` diff --git a/cmd/get_logs/main.go b/cmd/get_logs/main.go index 465349f9..3e013882 100644 --- a/cmd/get_logs/main.go +++ b/cmd/get_logs/main.go @@ -9,6 +9,8 @@ import ( "time" + "strings" + "github.com/8thlight/vulcanizedb/cmd" "github.com/8thlight/vulcanizedb/pkg/core" "github.com/8thlight/vulcanizedb/pkg/geth" @@ -29,11 +31,12 @@ const ( func main() { environment := flag.String("environment", "", "Environment name") contractHash := flag.String("contract-hash", "", "Contract hash to show summary") + flag.Parse() + ticker := time.NewTicker(pollingInterval) defer ticker.Stop() - flag.Parse() - + contractHashLowered := strings.ToLower(*contractHash) config := cmd.LoadConfig(*environment) blockchain := geth.NewBlockchain(config.Client.IPCPath) repository := cmd.LoadPostgres(config.Database, blockchain.Node()) @@ -43,7 +46,7 @@ func main() { go func() { for i := int64(0); i < lastBlockNumber; i = min(i+stepSize, lastBlockNumber) { - logs, err := blockchain.GetLogs(core.Contract{Hash: *contractHash}, big.NewInt(i), big.NewInt(i+stepSize)) + logs, err := blockchain.GetLogs(core.Contract{Hash: contractHashLowered}, big.NewInt(i), big.NewInt(i+stepSize)) log.Println("Backfilling Logs:", i) if err != nil { log.Println(err) @@ -61,7 +64,7 @@ func main() { z := &big.Int{} z.Sub(blockchain.LastBlock(), big.NewInt(25)) log.Printf("Logs Window: %d - %d", z.Int64(), blockchain.LastBlock().Int64()) - logs, _ := blockchain.GetLogs(core.Contract{Hash: *contractHash}, z, blockchain.LastBlock()) + logs, _ := blockchain.GetLogs(core.Contract{Hash: contractHashLowered}, z, blockchain.LastBlock()) repository.CreateLogs(logs) done <- struct{}{} }() diff --git a/cmd/run/main.go b/cmd/run/main.go index 2fd84573..a8fe9152 100644 --- a/cmd/run/main.go +++ b/cmd/run/main.go @@ -17,11 +17,12 @@ const ( ) func main() { + environment := flag.String("environment", "", "Environment name") + flag.Parse() + ticker := time.NewTicker(pollingInterval) defer ticker.Stop() - environment := flag.String("environment", "", "Environment name") - flag.Parse() config := cmd.LoadConfig(*environment) blockchain := geth.NewBlockchain(config.Client.IPCPath) repository := cmd.LoadPostgres(config.Database, blockchain.Node()) diff --git a/cmd/show_contract_summary/main.go b/cmd/show_contract_summary/main.go index 2b2a5e52..23ff73d1 100644 --- a/cmd/show_contract_summary/main.go +++ b/cmd/show_contract_summary/main.go @@ -7,6 +7,8 @@ import ( "fmt" + "strings" + "github.com/8thlight/vulcanizedb/cmd" "github.com/8thlight/vulcanizedb/pkg/contract_summary" "github.com/8thlight/vulcanizedb/pkg/geth" @@ -17,12 +19,14 @@ func main() { contractHash := flag.String("contract-hash", "", "Contract hash to show summary") _blockNumber := flag.Int64("block-number", -1, "Block number of summary") flag.Parse() + + contractHashLowered := strings.ToLower(*contractHash) config := cmd.LoadConfig(*environment) blockchain := geth.NewBlockchain(config.Client.IPCPath) repository := cmd.LoadPostgres(config.Database, blockchain.Node()) blockNumber := cmd.RequestedBlockNumber(_blockNumber) - contractSummary, err := contract_summary.NewSummary(blockchain, repository, *contractHash, blockNumber) + contractSummary, err := contract_summary.NewSummary(blockchain, repository, contractHashLowered, blockNumber) if err != nil { log.Fatalln(err) } diff --git a/cmd/utils.go b/cmd/utils.go index f4963b04..c7e5f8db 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -5,8 +5,6 @@ import ( "path/filepath" - "fmt" - "math/big" "github.com/8thlight/vulcanizedb/pkg/config" @@ -42,13 +40,14 @@ func ReadAbiFile(abiFilepath string) string { return abi } -func GetAbi(abiFilepath string, contractHash string) string { +func GetAbi(abiFilepath string, contractHash string, network string) string { var contractAbiString string if abiFilepath != "" { contractAbiString = ReadAbiFile(abiFilepath) } else { - etherscan := geth.NewEtherScanClient("https://api.etherscan.io") - fmt.Println("No ABI supplied. Retrieving ABI from Etherscan") + url := geth.GenUrl(network) + etherscan := geth.NewEtherScanClient(url) + log.Printf("No ABI supplied. Retrieving ABI from Etherscan: %s", url) contractAbiString, _ = etherscan.GetAbi(contractHash) } _, err := geth.ParseAbi(contractAbiString) diff --git a/cmd/vulcanize_db/main.go b/cmd/vulcanize_db/main.go index 1b5ee7be..b74be6b5 100644 --- a/cmd/vulcanize_db/main.go +++ b/cmd/vulcanize_db/main.go @@ -25,11 +25,12 @@ func backFillAllBlocks(blockchain core.Blockchain, repository repositories.Postg } func main() { + environment := flag.String("environment", "", "Environment name") + flag.Parse() + ticker := time.NewTicker(pollingInterval) defer ticker.Stop() - environment := flag.String("environment", "", "Environment name") - flag.Parse() config := cmd.LoadConfig(*environment) blockchain := geth.NewBlockchain(config.Client.IPCPath) repository := cmd.LoadPostgres(config.Database, blockchain.Node()) diff --git a/cmd/watch_contract/main.go b/cmd/watch_contract/main.go index 5bd35cb5..0a29f552 100644 --- a/cmd/watch_contract/main.go +++ b/cmd/watch_contract/main.go @@ -3,6 +3,8 @@ package main import ( "flag" + "strings" + "github.com/8thlight/vulcanizedb/cmd" "github.com/8thlight/vulcanizedb/pkg/core" "github.com/8thlight/vulcanizedb/pkg/geth" @@ -12,15 +14,18 @@ func main() { environment := flag.String("environment", "", "Environment name") contractHash := flag.String("contract-hash", "", "contract-hash=x1234") abiFilepath := flag.String("abi-filepath", "", "path/to/abifile.json") - flag.Parse() + network := flag.String("network", "", "ropsten") - contractAbiString := cmd.GetAbi(*abiFilepath, *contractHash) + flag.Parse() + contractHashLowered := strings.ToLower(*contractHash) + + contractAbiString := cmd.GetAbi(*abiFilepath, contractHashLowered, *network) config := cmd.LoadConfig(*environment) blockchain := geth.NewBlockchain(config.Client.IPCPath) repository := cmd.LoadPostgres(config.Database, blockchain.Node()) watchedContract := core.Contract{ Abi: contractAbiString, - Hash: *contractHash, + Hash: contractHashLowered, } repository.CreateContract(watchedContract) } diff --git a/pkg/geth/abi.go b/pkg/geth/abi.go index e84523fb..51b72103 100644 --- a/pkg/geth/abi.go +++ b/pkg/geth/abi.go @@ -38,6 +38,19 @@ func NewEtherScanClient(url string) *EtherScanApi { } +func GenUrl(network string) string { + switch network { + case "ropsten": + return "https://ropsten.etherscan.io" + case "kovan": + return "https://kovan.etherscan.io" + case "rinkeby": + return "https://rinkeby.etherscan.io" + default: + return "https://api.etherscan.io" + } +} + //https://api.etherscan.io/api?module=contract&action=getabi&address=%s func (e *EtherScanApi) GetAbi(contractHash string) (string, error) { target := new(Response) diff --git a/pkg/geth/abi_test.go b/pkg/geth/abi_test.go index 6a89faa9..da3391b1 100644 --- a/pkg/geth/abi_test.go +++ b/pkg/geth/abi_test.go @@ -101,5 +101,22 @@ var _ = Describe("ABI files", func() { }) }) }) + + Describe("Generating etherscan endpoints based on network", func() { + It("should return the main endpoint as the default", func() { + url := geth.GenUrl("") + Expect(url).To(Equal("https://api.etherscan.io")) + }) + + It("generates various test network endpoint if test network is supplied", func() { + ropstenUrl := geth.GenUrl("ropsten") + rinkebyUrl := geth.GenUrl("rinkeby") + kovanUrl := geth.GenUrl("kovan") + + Expect(ropstenUrl).To(Equal("https://ropsten.etherscan.io")) + Expect(kovanUrl).To(Equal("https://kovan.etherscan.io")) + Expect(rinkebyUrl).To(Equal("https://rinkeby.etherscan.io")) + }) + }) }) }) diff --git a/pkg/geth/block_to_core_block.go b/pkg/geth/block_to_core_block.go index 34df1660..8cc806e8 100644 --- a/pkg/geth/block_to_core_block.go +++ b/pkg/geth/block_to_core_block.go @@ -25,7 +25,7 @@ func ToCoreBlock(gethBlock *types.Block, client GethClient) core.Block { GasLimit: gethBlock.GasLimit().Int64(), GasUsed: gethBlock.GasUsed().Int64(), Hash: gethBlock.Hash().Hex(), - Miner: gethBlock.Coinbase().Hex(), + Miner: strings.ToLower(gethBlock.Coinbase().Hex()), Nonce: hexutil.Encode(gethBlock.Header().Nonce[:]), Number: gethBlock.Number().Int64(), ParentHash: gethBlock.ParentHash().Hex(), From f41bf49b0e3a5f5913d8f5b3e5caa5a0b1c4ba8b Mon Sep 17 00:00:00 2001 From: Matt K <1036969+mkrump@users.noreply.github.com> Date: Tue, 9 Jan 2018 08:45:15 -0600 Subject: [PATCH 04/22] Update README.md --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9ff7ea88..832c00ab 100644 --- a/README.md +++ b/README.md @@ -86,8 +86,9 @@ vulcanizedb/ .json private/ ``` -The name of the JSON file should correspond the contract's address. -2. Start watching the contract `godo watchContract -- --environment= --contract-hash=` +The name of the JSON file should correspond the contract's address. + +2. Start watching the contract `godo watchContract -- --environment= --contract-hash=` 3. Request summary data `godo showContractSummary -- --environment= --contract-hash=` @@ -115,4 +116,4 @@ You can create configuration files for additional environments. In order to run the integration tests, you will need to run them against a real blockchain. At the moment the integration tests require [Geth v1.7.2](https://ethereum.github.io/go-ethereum/downloads/) as they depend on the `--dev` mode, which changed in v1.7.3 1. Run `./scripts/start_private_blockchain` as a separate process. -2. `go test ./...` to run all tests. \ No newline at end of file +2. `go test ./...` to run all tests. From a9bea4f49210ac4ba6da132d0fe379ea1bc9fbb7 Mon Sep 17 00:00:00 2001 From: Matt K <1036969+mkrump@users.noreply.github.com> Date: Wed, 10 Jan 2018 15:54:36 -0600 Subject: [PATCH 05/22] Allow for multiple Geth nodes (#128) --- .../1515613181_add_fields_to_node.down.sql | 3 ++ .../1515613181_add_fields_to_node.up.sql | 3 ++ .../1515613715_update_node_index.down.sql | 9 ++++ .../1515613715_update_node_index.up.sql | 9 ++++ integration_test/geth_blockchain_test.go | 2 + pkg/core/node_info.go | 2 + pkg/fakes/blockchain.go | 2 +- pkg/geth/blockchain.go | 15 +++++- pkg/geth/node/node.go | 47 ++++++++++++++----- pkg/repositories/postgres.go | 13 +++-- pkg/repositories/postgres_test.go | 10 ++-- pkg/repositories/testing/helpers.go | 9 +++- 12 files changed, 98 insertions(+), 26 deletions(-) create mode 100644 db/migrations/1515613181_add_fields_to_node.down.sql create mode 100644 db/migrations/1515613181_add_fields_to_node.up.sql create mode 100644 db/migrations/1515613715_update_node_index.down.sql create mode 100644 db/migrations/1515613715_update_node_index.up.sql diff --git a/db/migrations/1515613181_add_fields_to_node.down.sql b/db/migrations/1515613181_add_fields_to_node.down.sql new file mode 100644 index 00000000..66ffb290 --- /dev/null +++ b/db/migrations/1515613181_add_fields_to_node.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE nodes + DROP COLUMN node_id, + DROP COLUMN client_name; \ No newline at end of file diff --git a/db/migrations/1515613181_add_fields_to_node.up.sql b/db/migrations/1515613181_add_fields_to_node.up.sql new file mode 100644 index 00000000..5396e26a --- /dev/null +++ b/db/migrations/1515613181_add_fields_to_node.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE nodes + ADD COLUMN node_id VARCHAR(128), + ADD COLUMN client_name VARCHAR; \ No newline at end of file diff --git a/db/migrations/1515613715_update_node_index.down.sql b/db/migrations/1515613715_update_node_index.down.sql new file mode 100644 index 00000000..5c252c4b --- /dev/null +++ b/db/migrations/1515613715_update_node_index.down.sql @@ -0,0 +1,9 @@ +BEGIN; + +ALTER TABLE nodes + DROP CONSTRAINT node_uc; + +ALTER TABLE nodes + ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id); + +COMMIT; diff --git a/db/migrations/1515613715_update_node_index.up.sql b/db/migrations/1515613715_update_node_index.up.sql new file mode 100644 index 00000000..6abf9347 --- /dev/null +++ b/db/migrations/1515613715_update_node_index.up.sql @@ -0,0 +1,9 @@ +BEGIN; + +ALTER TABLE nodes + DROP CONSTRAINT node_uc; + +ALTER TABLE nodes + ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id); + +COMMIT; \ No newline at end of file diff --git a/integration_test/geth_blockchain_test.go b/integration_test/geth_blockchain_test.go index 279fbef5..c3a5b86d 100644 --- a/integration_test/geth_blockchain_test.go +++ b/integration_test/geth_blockchain_test.go @@ -52,6 +52,8 @@ var _ = Describe("Reading from the Geth blockchain", func() { Expect(node.GenesisBlock).To(Equal(devNetworkGenesisBlock)) Expect(node.NetworkId).To(Equal(devNetworkNodeId)) + Expect(len(node.Id)).To(Equal(128)) + Expect(node.ClientName).To(ContainSubstring("Geth")) close(done) }, 15) diff --git a/pkg/core/node_info.go b/pkg/core/node_info.go index 79ae55fb..fe88c928 100644 --- a/pkg/core/node_info.go +++ b/pkg/core/node_info.go @@ -3,4 +3,6 @@ package core type Node struct { GenesisBlock string NetworkId float64 + Id string + ClientName string } diff --git a/pkg/fakes/blockchain.go b/pkg/fakes/blockchain.go index 2af3a024..cdc4619a 100644 --- a/pkg/fakes/blockchain.go +++ b/pkg/fakes/blockchain.go @@ -50,7 +50,7 @@ func NewBlockchain() *Blockchain { blocks: make(map[int64]core.Block), logs: make(map[string][]core.Log), contractAttributes: make(map[string]map[string]string), - node: core.Node{GenesisBlock: "GENESIS"}, + node: core.Node{GenesisBlock: "GENESIS", NetworkId: 1, Id: "x123", ClientName: "Geth"}, } } diff --git a/pkg/geth/blockchain.go b/pkg/geth/blockchain.go index a7bba5f0..c519706f 100644 --- a/pkg/geth/blockchain.go +++ b/pkg/geth/blockchain.go @@ -3,6 +3,8 @@ package geth import ( "math/big" + "strings" + "github.com/8thlight/vulcanizedb/pkg/core" "github.com/8thlight/vulcanizedb/pkg/geth/node" "github.com/ethereum/go-ethereum" @@ -25,11 +27,22 @@ func NewBlockchain(ipcPath string) *Blockchain { blockchain := Blockchain{} rpcClient, _ := rpc.Dial(ipcPath) client := ethclient.NewClient(rpcClient) - blockchain.node = node.Retrieve(rpcClient) + blockchain.node = node.Info(rpcClient) + if infura := isInfuraNode(ipcPath); infura { + blockchain.node.Id = "infura" + blockchain.node.ClientName = "infura" + } blockchain.client = client return &blockchain } +func isInfuraNode(ipcPath string) bool { + if strings.Contains(ipcPath, "infura") { + return true + } + return false +} + func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]core.Log, error) { if endingBlockNumber == nil { endingBlockNumber = startingBlockNumber diff --git a/pkg/geth/node/node.go b/pkg/geth/node/node.go index 524316dd..71deb4a3 100644 --- a/pkg/geth/node/node.go +++ b/pkg/geth/node/node.go @@ -7,22 +7,43 @@ import ( "github.com/8thlight/vulcanizedb/pkg/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rpc" ) -func Retrieve(client *rpc.Client) core.Node { +func Info(client *rpc.Client) core.Node { node := core.Node{} - - var version string - client.CallContext(context.Background(), &version, "net_version") - node.NetworkId, _ = strconv.ParseFloat(version, 64) - - var protocolVersion string - client.CallContext(context.Background(), &protocolVersion, "eth_protocolVersion") - - var header *types.Header - client.CallContext(context.Background(), &header, "eth_getBlockByNumber", "0x0", false) - node.GenesisBlock = header.Hash().Hex() - + node.NetworkId = NetworkId(client) + node.GenesisBlock = GenesisBlock(client) + node.Id, node.ClientName = IdClientName(client) return node } + +func IdClientName(client *rpc.Client) (string, string) { + var info p2p.NodeInfo + modules, _ := client.SupportedModules() + if _, ok := modules["admin"]; ok { + client.CallContext(context.Background(), &info, "admin_nodeInfo") + return info.ID, info.Name + } + return "", "" +} + +func NetworkId(client *rpc.Client) float64 { + var version string + client.CallContext(context.Background(), &version, "net_version") + networkId, _ := strconv.ParseFloat(version, 64) + return networkId +} + +func ProtocolVersion(client *rpc.Client) string { + var protocolVersion string + client.CallContext(context.Background(), &protocolVersion, "eth_protocolVersion") + return protocolVersion +} + +func GenesisBlock(client *rpc.Client) string { + var header *types.Header + client.CallContext(context.Background(), &header, "eth_getBlockByNumber", "0x0", false) + return header.Hash().Hex() +} diff --git a/pkg/repositories/postgres.go b/pkg/repositories/postgres.go index de8b8128..4418073a 100644 --- a/pkg/repositories/postgres.go +++ b/pkg/repositories/postgres.go @@ -111,13 +111,16 @@ func (repository Postgres) FindLogs(address string, blockNumber int64) []core.Lo func (repository *Postgres) CreateNode(node *core.Node) error { var nodeId int64 err := repository.Db.QueryRow( - `INSERT INTO nodes (genesis_block, network_id) - VALUES ($1, $2) - ON CONFLICT (genesis_block, network_id) + `INSERT INTO nodes (genesis_block, network_id, node_id, client_name) + VALUES ($1, $2, $3, $4) + ON CONFLICT (genesis_block, network_id, node_id) DO UPDATE - SET genesis_block = $1, network_id = $2 + SET genesis_block = $1, + network_id = $2, + node_id = $3, + client_name = $4 RETURNING id`, - node.GenesisBlock, node.NetworkId).Scan(&nodeId) + node.GenesisBlock, node.NetworkId, node.Id, node.ClientName).Scan(&nodeId) if err != nil { return ErrUnableToSetNode } diff --git a/pkg/repositories/postgres_test.go b/pkg/repositories/postgres_test.go index 7768d907..ce75c2c5 100644 --- a/pkg/repositories/postgres_test.go +++ b/pkg/repositories/postgres_test.go @@ -47,7 +47,7 @@ var _ = Describe("Postgres repository", func() { Transactions: []core.Transaction{}, } cfg, _ := config.NewConfig("private") - node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1} + node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1, Id: "x123", ClientName: "geth"} repository, _ := repositories.NewPostgres(cfg.Database, node) err1 := repository.CreateOrUpdateBlock(badBlock) @@ -60,7 +60,7 @@ var _ = Describe("Postgres repository", func() { It("throws error when can't connect to the database", func() { invalidDatabase := config.Database{} - node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1} + node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1, Id: "x123", ClientName: "geth"} _, err := repositories.NewPostgres(invalidDatabase, node) Expect(err).To(Equal(repositories.ErrDBConnectionFailed)) }) @@ -68,7 +68,7 @@ var _ = Describe("Postgres repository", func() { It("throws error when can't create node", func() { cfg, _ := config.NewConfig("private") badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100)) - node := core.Node{GenesisBlock: badHash, NetworkId: 1} + node := core.Node{GenesisBlock: badHash, NetworkId: 1, Id: "x123", ClientName: "geth"} _, err := repositories.NewPostgres(cfg.Database, node) Expect(err).To(Equal(repositories.ErrUnableToSetNode)) }) @@ -82,7 +82,7 @@ var _ = Describe("Postgres repository", func() { TxHash: badTxHash, } cfg, _ := config.NewConfig("private") - node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1} + node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1, Id: "x123", ClientName: "geth"} repository, _ := repositories.NewPostgres(cfg.Database, node) err := repository.CreateLogs([]core.Log{badLog}) @@ -101,7 +101,7 @@ var _ = Describe("Postgres repository", func() { Transactions: []core.Transaction{badTransaction}, } cfg, _ := config.NewConfig("private") - node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1} + node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1, Id: "x123", ClientName: "geth"} repository, _ := repositories.NewPostgres(cfg.Database, node) err1 := repository.CreateOrUpdateBlock(block) diff --git a/pkg/repositories/testing/helpers.go b/pkg/repositories/testing/helpers.go index 784217b1..f2b42138 100644 --- a/pkg/repositories/testing/helpers.go +++ b/pkg/repositories/testing/helpers.go @@ -22,7 +22,12 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. var repository repositories.Repository BeforeEach(func() { - node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1} + node := core.Node{ + GenesisBlock: "GENESIS", + NetworkId: 1, + Id: "b6f90c0fdd8ec9607aed8ee45c69322e47b7063f0bfb7a29c8ecafab24d0a22d24dd2329b5ee6ed4125a03cb14e57fd584e67f9e53e6c631055cbbd82f080845", + ClientName: "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9", + } repository = buildRepository(node) }) @@ -48,6 +53,8 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. nodeTwo := core.Node{ GenesisBlock: "0x456", NetworkId: 1, + Id: "x123456", + ClientName: "Geth", } repositoryTwo := buildRepository(nodeTwo) From 431be46005237aa73a26489b565554759824cfc3 Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Mon, 15 Jan 2018 10:52:50 -0600 Subject: [PATCH 06/22] Add store logs with receipts --- pkg/repositories/in_memory.go | 1 + pkg/repositories/postgres.go | 3 ++ pkg/repositories/testing/helpers.go | 81 +++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) diff --git a/pkg/repositories/in_memory.go b/pkg/repositories/in_memory.go index 58cf4fe1..0a609231 100644 --- a/pkg/repositories/in_memory.go +++ b/pkg/repositories/in_memory.go @@ -102,6 +102,7 @@ func (repository *InMemory) CreateOrUpdateBlock(block core.Block) error { repository.blocks[block.Number] = block for _, transaction := range block.Transactions { repository.receipts[transaction.Hash] = transaction.Receipt + repository.logs[transaction.TxHash] = transaction.Logs } return nil } diff --git a/pkg/repositories/postgres.go b/pkg/repositories/postgres.go index 4418073a..d7d82402 100644 --- a/pkg/repositories/postgres.go +++ b/pkg/repositories/postgres.go @@ -344,6 +344,9 @@ func (repository Postgres) createTransaction(tx *sql.Tx, blockId int64, transact if err != nil { return err } + if len(transaction.Receipt.Logs) > 0 { + err = repository.CreateLogs(transaction.Receipt.Logs) + } } return nil } diff --git a/pkg/repositories/testing/helpers.go b/pkg/repositories/testing/helpers.go index f2b42138..2ce90b3d 100644 --- a/pkg/repositories/testing/helpers.go +++ b/pkg/repositories/testing/helpers.go @@ -504,6 +504,87 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. {blockNumber: 1, Index: 1}}, )) }) + + It("saves the logs attached to a receipt", func() { + logs := []core.Log{{ + Address: "0x8a4774fe82c63484afef97ca8d89a6ea5e21f973", + BlockNumber: 4745407, + Data: "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000645a68669900000000000000000000000000000000000000000000003397684ab5869b0000000000000000000000000000000000000000000000000000000000005a36053200000000000000000000000099041f808d598b782d5a3e498681c2452a31da08", + Index: 86, + Topics: map[int]string{ + 0: "0x5a68669900000000000000000000000000000000000000000000000000000000", + 1: "0x000000000000000000000000d0148dad63f73ce6f1b6c607e3413dcf1ff5f030", + 2: "0x00000000000000000000000000000000000000000000003397684ab5869b0000", + 3: "0x000000000000000000000000000000000000000000000000000000005a360532", + }, + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + }, { + Address: "0x99041f808d598b782d5a3e498681c2452a31da08", + BlockNumber: 4745407, + Data: "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000418178358", + Index: 87, + Topics: map[int]string{ + 0: "0x1817835800000000000000000000000000000000000000000000000000000000", + 1: "0x0000000000000000000000008a4774fe82c63484afef97ca8d89a6ea5e21f973", + 2: "0x0000000000000000000000000000000000000000000000000000000000000000", + 3: "0x0000000000000000000000000000000000000000000000000000000000000000", + }, + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + }, { + Address: "0x99041f808d598b782d5a3e498681c2452a31da08", + BlockNumber: 4745407, + Data: "0x00000000000000000000000000000000000000000000003338f64c8423af4000", + Index: 88, + Topics: map[int]string{ + 0: "0x296ba4ca62c6c21c95e828080cb8aec7481b71390585605300a8a76f9e95b527", + }, + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + }, + } + receipt := core.Receipt{ + ContractAddress: "", + CumulativeGasUsed: 7481414, + GasUsed: 60711, + Logs: logs, + Bloom: "0x00000800000000000000001000000000000000400000000080000000000000000000400000010000000000000000000000000000040000000000000002000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000800004000000000000001000000000000000000000000000002000000480000000000000002000000000000000020000000000000000000000000000000000000000080000000000180000c00000000000000002000002000000040000000000000000000000000000010000000000020000000000000000000002000000000000000000000000400800000000000000000", + Status: 1, + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + } + transaction := + core.Transaction{ + Hash: receipt.TxHash, + Receipt: receipt, + } + + block := core.Block{Transactions: []core.Transaction{transaction}} + repository.CreateOrUpdateBlock(block) + retrievedLogs := repository.FindLogs("0x99041f808d598b782d5a3e498681c2452a31da08", 4745407) + + Expect(retrievedLogs).To(Not(BeZero())) + }) + + It("saves the logs attached to a receipt", func() { + logs := make([]core.Log, 0) + receipt := core.Receipt{ + Logs: logs, + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + } + transaction := core.Transaction{ + Hash: receipt.TxHash, + Receipt: receipt, + } + + block := core.Block{ + Transactions: []core.Transaction{transaction}, + } + repository.CreateOrUpdateBlock(block) + + retrievedLogs := repository.FindLogs("0x99041f808d598b782d5a3e498681c2452a31da08", 4745407) + _, err := repository.FindReceipt(receipt.TxHash) + + Expect(err).To(Not(HaveOccurred())) + Expect(retrievedLogs).To(BeZero()) + }) }) Describe("Saving receipts", func() { From 50f00b80c1bd978a1f23fe92861eef0d73087d52 Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Mon, 15 Jan 2018 11:21:08 -0600 Subject: [PATCH 07/22] Lowercase log address --- pkg/geth/log_to_core_log.go | 4 +++- pkg/geth/log_to_core_log_test.go | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/geth/log_to_core_log.go b/pkg/geth/log_to_core_log.go index f40983f4..bce6fd4c 100644 --- a/pkg/geth/log_to_core_log.go +++ b/pkg/geth/log_to_core_log.go @@ -1,6 +1,8 @@ package geth import ( + "strings" + "github.com/8thlight/vulcanizedb/pkg/core" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -13,7 +15,7 @@ func LogToCoreLog(gethLog types.Log) core.Log { hexTopics[i] = topic.Hex() } return core.Log{ - Address: gethLog.Address.Hex(), + Address: strings.ToLower(gethLog.Address.Hex()), BlockNumber: int64(gethLog.BlockNumber), Topics: hexTopics, diff --git a/pkg/geth/log_to_core_log_test.go b/pkg/geth/log_to_core_log_test.go index 44448ce3..ad15e826 100644 --- a/pkg/geth/log_to_core_log_test.go +++ b/pkg/geth/log_to_core_log_test.go @@ -1,6 +1,8 @@ package geth_test import ( + "strings" + "github.com/8thlight/vulcanizedb/pkg/core" "github.com/8thlight/vulcanizedb/pkg/geth" "github.com/ethereum/go-ethereum/common" @@ -14,7 +16,7 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { It("converts geth log to internal log format", func() { gethLog := types.Log{ - Address: common.HexToAddress("0xecf8f87f810ecf450940c9f60066b4a7a501d6a7"), + Address: common.HexToAddress("0x448a5065aeBB8E423F0896E6c5D525C040f59af3"), BlockHash: common.HexToHash("0x656c34545f90a730a19008c0e7a7cd4fb3895064b48d6d69761bd5abad681056"), BlockNumber: 2019236, Data: hexutil.MustDecode("0x000000000000000000000000000000000000000000000001a055690d9db80000"), @@ -28,7 +30,7 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { } expected := core.Log{ - Address: gethLog.Address.Hex(), + Address: strings.ToLower(gethLog.Address.Hex()), BlockNumber: int64(gethLog.BlockNumber), Data: hexutil.Encode(gethLog.Data), TxHash: gethLog.TxHash.Hex(), From 9ee13e715dbe2b427032b5ce128ccd621a0fa647 Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Mon, 15 Jan 2018 14:44:47 -0600 Subject: [PATCH 08/22] Add topics type --- integration_test/contract_test.go | 4 +- pkg/core/log.go | 7 +-- pkg/core/topics.go | 3 ++ pkg/geth/blockchain.go | 2 +- pkg/geth/log_to_core_log.go | 31 ++++++++----- pkg/geth/log_to_core_log_test.go | 7 ++- pkg/repositories/postgres.go | 69 +++++++++++++++-------------- pkg/repositories/testing/helpers.go | 21 ++++----- 8 files changed, 80 insertions(+), 64 deletions(-) create mode 100644 pkg/core/topics.go diff --git a/integration_test/contract_test.go b/integration_test/contract_test.go index 6d0af83f..33784973 100644 --- a/integration_test/contract_test.go +++ b/integration_test/contract_test.go @@ -107,8 +107,8 @@ var _ = Describe("Reading contracts", func() { expectedLogZero := core.Log{ BlockNumber: 4703824, TxHash: "0xf896bfd1eb539d881a1a31102b78de9f25cd591bf1fe1924b86148c0b205fd5d", - Address: "0xd26114cd6EE289AccF82350c8d8487fedB8A0C07", - Topics: map[int]string{ + Address: "0xd26114cd6ee289accf82350c8d8487fedb8a0c07", + Topics: core.Topics{ 0: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", 1: "0x000000000000000000000000fbb1b73c4f0bda4f67dca266ce6ef42f520fbb98", 2: "0x000000000000000000000000d26114cd6ee289accf82350c8d8487fedb8a0c07", diff --git a/pkg/core/log.go b/pkg/core/log.go index 9688b46c..c96817f9 100644 --- a/pkg/core/log.go +++ b/pkg/core/log.go @@ -4,7 +4,8 @@ type Log struct { BlockNumber int64 TxHash string Address string - Topics map[int]string - Index int64 - Data string + Topics + Index int64 + Data string + Removed bool } diff --git a/pkg/core/topics.go b/pkg/core/topics.go new file mode 100644 index 00000000..2f1acb9d --- /dev/null +++ b/pkg/core/topics.go @@ -0,0 +1,3 @@ +package core + +type Topics [4]string diff --git a/pkg/geth/blockchain.go b/pkg/geth/blockchain.go index c519706f..aad367d3 100644 --- a/pkg/geth/blockchain.go +++ b/pkg/geth/blockchain.go @@ -57,7 +57,7 @@ func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumbe if err != nil { return []core.Log{}, err } - logs := GethLogsToCoreLogs(gethLogs) + logs := LogsToCoreLogs(gethLogs) return logs, nil } diff --git a/pkg/geth/log_to_core_log.go b/pkg/geth/log_to_core_log.go index bce6fd4c..f3bc3534 100644 --- a/pkg/geth/log_to_core_log.go +++ b/pkg/geth/log_to_core_log.go @@ -4,16 +4,31 @@ import ( "strings" "github.com/8thlight/vulcanizedb/pkg/core" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" ) -func LogToCoreLog(gethLog types.Log) core.Log { - topics := gethLog.Topics - var hexTopics = make(map[int]string) +func LogsToCoreLogs(gethLogs []types.Log) []core.Log { + var logs []core.Log + for _, log := range gethLogs { + log := LogToCoreLog(log) + logs = append(logs, log) + } + return logs +} + +func MakeTopics(topics []common.Hash) core.Topics { + var hexTopics core.Topics for i, topic := range topics { hexTopics[i] = topic.Hex() } + return hexTopics +} + +func LogToCoreLog(gethLog types.Log) core.Log { + topics := gethLog.Topics + hexTopics := MakeTopics(topics) return core.Log{ Address: strings.ToLower(gethLog.Address.Hex()), @@ -22,14 +37,6 @@ func LogToCoreLog(gethLog types.Log) core.Log { TxHash: gethLog.TxHash.Hex(), Index: int64(gethLog.Index), Data: hexutil.Encode(gethLog.Data), + Removed: gethLog.Removed, } } - -func GethLogsToCoreLogs(gethLogs []types.Log) []core.Log { - var logs []core.Log - for _, log := range gethLogs { - log := LogToCoreLog(log) - logs = append(logs, log) - } - return logs -} diff --git a/pkg/geth/log_to_core_log_test.go b/pkg/geth/log_to_core_log_test.go index ad15e826..a470b91c 100644 --- a/pkg/geth/log_to_core_log_test.go +++ b/pkg/geth/log_to_core_log_test.go @@ -27,6 +27,7 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"), common.HexToHash("0x00000000000000000000000080b2c9d7cbbf30a1b0fc8983c647d754c6525615"), }, + Removed: true, } expected := core.Log{ @@ -35,10 +36,11 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { Data: hexutil.Encode(gethLog.Data), TxHash: gethLog.TxHash.Hex(), Index: 2, - Topics: map[int]string{ + Topics: core.Topics{ 0: common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef").Hex(), 1: common.HexToHash("0x00000000000000000000000080b2c9d7cbbf30a1b0fc8983c647d754c6525615").Hex(), }, + Removed: gethLog.Removed, } coreLog := geth.LogToCoreLog(gethLog) @@ -46,6 +48,7 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { Expect(coreLog.Address).To(Equal(expected.Address)) Expect(coreLog.BlockNumber).To(Equal(expected.BlockNumber)) Expect(coreLog.Data).To(Equal(expected.Data)) + Expect(coreLog.Removed).To(Equal(expected.Removed)) Expect(coreLog.Index).To(Equal(expected.Index)) Expect(coreLog.Topics[0]).To(Equal(expected.Topics[0])) Expect(coreLog.Topics[1]).To(Equal(expected.Topics[1])) @@ -84,7 +87,7 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { expectedOne := geth.LogToCoreLog(gethLogOne) expectedTwo := geth.LogToCoreLog(gethLogTwo) - coreLogs := geth.GethLogsToCoreLogs([]types.Log{gethLogOne, gethLogTwo}) + coreLogs := geth.LogsToCoreLogs([]types.Log{gethLogOne, gethLogTwo}) Expect(len(coreLogs)).To(Equal(2)) Expect(coreLogs[0]).To(Equal(expectedOne)) diff --git a/pkg/repositories/postgres.go b/pkg/repositories/postgres.go index d7d82402..5461fe53 100644 --- a/pkg/repositories/postgres.go +++ b/pkg/repositories/postgres.go @@ -62,35 +62,6 @@ func (repository Postgres) SetBlocksStatus(chainHead int64) { cutoff) } -func (repository Postgres) CreateLogs(logs []core.Log) error { - tx, _ := repository.Db.BeginTx(context.Background(), nil) - for _, tlog := range logs { - _, err := tx.Exec( - `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ON CONFLICT (index, block_number) - DO UPDATE - SET block_number = $1, - address = $2, - tx_hash = $3, - index = $4, - topic0 = $5, - topic1 = $6, - topic2 = $7, - topic3 = $8, - data = $9 - `, - tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, - ) - if err != nil { - tx.Rollback() - return ErrDBInsertFailed - } - } - tx.Commit() - return nil -} - func (repository Postgres) FindLogs(address string, blockNumber int64) []core.Log { logRows, _ := repository.Db.Query( `SELECT block_number, @@ -353,17 +324,48 @@ func (repository Postgres) createTransaction(tx *sql.Tx, blockId int64, transact func (repository Postgres) createReceipt(tx *sql.Tx, transactionId int, receipt core.Receipt) error { //Not currently persisting log bloom filters - _, err := tx.Exec( + var receiptId int + err := tx.QueryRow( `INSERT INTO receipts (contract_address, tx_hash, cumulative_gas_used, gas_used, state_root, status, transaction_id) - VALUES ($1, $2, $3, $4, $5, $6, $7)`, - receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, transactionId) + VALUES ($1, $2, $3, $4, $5, $6, $7) + RETURNING id`, + receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, transactionId).Scan(&receiptId) if err != nil { return err } return nil } +func (repository Postgres) CreateLogs(logs []core.Log) error { + tx, _ := repository.Db.BeginTx(context.Background(), nil) + for _, tlog := range logs { + _, err := tx.Exec( + `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (index, block_number) + DO UPDATE + SET block_number = $1, + address = $2, + tx_hash = $3, + index = $4, + topic0 = $5, + topic1 = $6, + topic2 = $7, + topic3 = $8, + data = $9 + `, + tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, + ) + if err != nil { + tx.Rollback() + return ErrDBInsertFailed + } + } + tx.Commit() + return nil +} + func loadReceipt(receiptsRow *sql.Row) (core.Receipt, error) { var contractAddress string var txHash string @@ -445,7 +447,7 @@ func (repository Postgres) loadLogs(logsRows *sql.Rows) []core.Log { var txHash string var index int64 var data string - topics := make([]string, 4) + var topics core.Topics logsRows.Scan(&blockNumber, &address, &txHash, &index, &topics[0], &topics[1], &topics[2], &topics[3], &data) log := core.Log{ BlockNumber: blockNumber, @@ -454,7 +456,6 @@ func (repository Postgres) loadLogs(logsRows *sql.Rows) []core.Log { Index: index, Data: data, } - log.Topics = make(map[int]string) for i, topic := range topics { log.Topics[i] = topic } diff --git a/pkg/repositories/testing/helpers.go b/pkg/repositories/testing/helpers.go index 2ce90b3d..251d043d 100644 --- a/pkg/repositories/testing/helpers.go +++ b/pkg/repositories/testing/helpers.go @@ -399,7 +399,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, Data: "xabc", }}, ) @@ -428,7 +428,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, Data: "xABC", }, }) @@ -437,7 +437,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, Data: "xXYZ", }, }) @@ -452,7 +452,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, Data: "xabc", }}, ) @@ -461,7 +461,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 1, Address: "x123", TxHash: "x789", - Topics: map[int]string{0: "x111", 1: "x222", 2: "x333"}, + Topics: [4]string{0: "x111", 1: "x222", 2: "x333"}, Data: "xdef", }}, ) @@ -470,7 +470,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, Data: "xabc", }}, ) @@ -511,7 +511,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. BlockNumber: 4745407, Data: "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000645a68669900000000000000000000000000000000000000000000003397684ab5869b0000000000000000000000000000000000000000000000000000000000005a36053200000000000000000000000099041f808d598b782d5a3e498681c2452a31da08", Index: 86, - Topics: map[int]string{ + Topics: core.Topics{ 0: "0x5a68669900000000000000000000000000000000000000000000000000000000", 1: "0x000000000000000000000000d0148dad63f73ce6f1b6c607e3413dcf1ff5f030", 2: "0x00000000000000000000000000000000000000000000003397684ab5869b0000", @@ -523,7 +523,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. BlockNumber: 4745407, Data: "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000418178358", Index: 87, - Topics: map[int]string{ + Topics: core.Topics{ 0: "0x1817835800000000000000000000000000000000000000000000000000000000", 1: "0x0000000000000000000000008a4774fe82c63484afef97ca8d89a6ea5e21f973", 2: "0x0000000000000000000000000000000000000000000000000000000000000000", @@ -535,7 +535,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. BlockNumber: 4745407, Data: "0x00000000000000000000000000000000000000000000003338f64c8423af4000", Index: 88, - Topics: map[int]string{ + Topics: core.Topics{ 0: "0x296ba4ca62c6c21c95e828080cb8aec7481b71390585605300a8a76f9e95b527", }, TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", @@ -557,7 +557,8 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. } block := core.Block{Transactions: []core.Transaction{transaction}} - repository.CreateOrUpdateBlock(block) + err := repository.CreateOrUpdateBlock(block) + Expect(err).To(Not(HaveOccurred())) retrievedLogs := repository.FindLogs("0x99041f808d598b782d5a3e498681c2452a31da08", 4745407) Expect(retrievedLogs).To(Not(BeZero())) From 82c39a2c1febeaf4b64590cc5741af9292006570 Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Mon, 15 Jan 2018 15:27:45 -0600 Subject: [PATCH 09/22] Add fk constraint on logs --- .../1516050071_add_log_fk_constraint.down.sql | 9 ++++ .../1516050071_add_log_fk_constraint.up.sql | 12 +++++ pkg/core/log.go | 5 +- pkg/geth/blockchain.go | 9 +++- pkg/geth/log_to_core_log.go | 11 ++--- pkg/geth/log_to_core_log_test.go | 15 +++--- pkg/geth/receipt_to_core_receipt.go | 2 +- pkg/repositories/postgres.go | 49 ++++++++++++++++--- pkg/repositories/testing/helpers.go | 21 ++++---- 9 files changed, 94 insertions(+), 39 deletions(-) create mode 100644 db/migrations/1516050071_add_log_fk_constraint.down.sql create mode 100644 db/migrations/1516050071_add_log_fk_constraint.up.sql diff --git a/db/migrations/1516050071_add_log_fk_constraint.down.sql b/db/migrations/1516050071_add_log_fk_constraint.down.sql new file mode 100644 index 00000000..6ce8446f --- /dev/null +++ b/db/migrations/1516050071_add_log_fk_constraint.down.sql @@ -0,0 +1,9 @@ +BEGIN; + +ALTER TABLE logs + DROP CONSTRAINT receipts_fk; + +ALTER TABLE logs + DROP COLUMN receipt_id; + +COMMIT; \ No newline at end of file diff --git a/db/migrations/1516050071_add_log_fk_constraint.up.sql b/db/migrations/1516050071_add_log_fk_constraint.up.sql new file mode 100644 index 00000000..a0150d1a --- /dev/null +++ b/db/migrations/1516050071_add_log_fk_constraint.up.sql @@ -0,0 +1,12 @@ +BEGIN; + +ALTER TABLE logs + ADD COLUMN receipt_id INT; + +ALTER TABLE logs + ADD CONSTRAINT receipts_fk +FOREIGN KEY (receipt_id) +REFERENCES receipts (id) +ON DELETE CASCADE; + +COMMIT; \ No newline at end of file diff --git a/pkg/core/log.go b/pkg/core/log.go index c96817f9..984e0432 100644 --- a/pkg/core/log.go +++ b/pkg/core/log.go @@ -5,7 +5,6 @@ type Log struct { TxHash string Address string Topics - Index int64 - Data string - Removed bool + Index int64 + Data string } diff --git a/pkg/geth/blockchain.go b/pkg/geth/blockchain.go index aad367d3..5ff500c2 100644 --- a/pkg/geth/blockchain.go +++ b/pkg/geth/blockchain.go @@ -5,6 +5,8 @@ import ( "strings" + "log" + "github.com/8thlight/vulcanizedb/pkg/core" "github.com/8thlight/vulcanizedb/pkg/geth/node" "github.com/ethereum/go-ethereum" @@ -25,7 +27,10 @@ type Blockchain struct { func NewBlockchain(ipcPath string) *Blockchain { blockchain := Blockchain{} - rpcClient, _ := rpc.Dial(ipcPath) + rpcClient, err := rpc.Dial(ipcPath) + if err != nil { + log.Fatal(err) + } client := ethclient.NewClient(rpcClient) blockchain.node = node.Info(rpcClient) if infura := isInfuraNode(ipcPath); infura { @@ -57,7 +62,7 @@ func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumbe if err != nil { return []core.Log{}, err } - logs := LogsToCoreLogs(gethLogs) + logs := ToCoreLogs(gethLogs) return logs, nil } diff --git a/pkg/geth/log_to_core_log.go b/pkg/geth/log_to_core_log.go index f3bc3534..7b24175f 100644 --- a/pkg/geth/log_to_core_log.go +++ b/pkg/geth/log_to_core_log.go @@ -9,16 +9,16 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) -func LogsToCoreLogs(gethLogs []types.Log) []core.Log { +func ToCoreLogs(gethLogs []types.Log) []core.Log { var logs []core.Log for _, log := range gethLogs { - log := LogToCoreLog(log) + log := ToCoreLog(log) logs = append(logs, log) } return logs } -func MakeTopics(topics []common.Hash) core.Topics { +func makeTopics(topics []common.Hash) core.Topics { var hexTopics core.Topics for i, topic := range topics { hexTopics[i] = topic.Hex() @@ -26,9 +26,9 @@ func MakeTopics(topics []common.Hash) core.Topics { return hexTopics } -func LogToCoreLog(gethLog types.Log) core.Log { +func ToCoreLog(gethLog types.Log) core.Log { topics := gethLog.Topics - hexTopics := MakeTopics(topics) + hexTopics := makeTopics(topics) return core.Log{ Address: strings.ToLower(gethLog.Address.Hex()), @@ -37,6 +37,5 @@ func LogToCoreLog(gethLog types.Log) core.Log { TxHash: gethLog.TxHash.Hex(), Index: int64(gethLog.Index), Data: hexutil.Encode(gethLog.Data), - Removed: gethLog.Removed, } } diff --git a/pkg/geth/log_to_core_log_test.go b/pkg/geth/log_to_core_log_test.go index a470b91c..000276ff 100644 --- a/pkg/geth/log_to_core_log_test.go +++ b/pkg/geth/log_to_core_log_test.go @@ -27,7 +27,6 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"), common.HexToHash("0x00000000000000000000000080b2c9d7cbbf30a1b0fc8983c647d754c6525615"), }, - Removed: true, } expected := core.Log{ @@ -37,18 +36,16 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { TxHash: gethLog.TxHash.Hex(), Index: 2, Topics: core.Topics{ - 0: common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef").Hex(), - 1: common.HexToHash("0x00000000000000000000000080b2c9d7cbbf30a1b0fc8983c647d754c6525615").Hex(), + gethLog.Topics[0].Hex(), + gethLog.Topics[1].Hex(), }, - Removed: gethLog.Removed, } - coreLog := geth.LogToCoreLog(gethLog) + coreLog := geth.ToCoreLog(gethLog) Expect(coreLog.Address).To(Equal(expected.Address)) Expect(coreLog.BlockNumber).To(Equal(expected.BlockNumber)) Expect(coreLog.Data).To(Equal(expected.Data)) - Expect(coreLog.Removed).To(Equal(expected.Removed)) Expect(coreLog.Index).To(Equal(expected.Index)) Expect(coreLog.Topics[0]).To(Equal(expected.Topics[0])) Expect(coreLog.Topics[1]).To(Equal(expected.Topics[1])) @@ -84,10 +81,10 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { }, } - expectedOne := geth.LogToCoreLog(gethLogOne) - expectedTwo := geth.LogToCoreLog(gethLogTwo) + expectedOne := geth.ToCoreLog(gethLogOne) + expectedTwo := geth.ToCoreLog(gethLogTwo) - coreLogs := geth.LogsToCoreLogs([]types.Log{gethLogOne, gethLogTwo}) + coreLogs := geth.ToCoreLogs([]types.Log{gethLogOne, gethLogTwo}) Expect(len(coreLogs)).To(Equal(2)) Expect(coreLogs[0]).To(Equal(expectedOne)) diff --git a/pkg/geth/receipt_to_core_receipt.go b/pkg/geth/receipt_to_core_receipt.go index b9130f4b..7117a6b7 100644 --- a/pkg/geth/receipt_to_core_receipt.go +++ b/pkg/geth/receipt_to_core_receipt.go @@ -49,7 +49,7 @@ func setContractAddress(gethReceipt *types.Receipt) string { func dereferenceLogs(gethReceipt *types.Receipt) []core.Log { logs := []core.Log{} for _, log := range gethReceipt.Logs { - logs = append(logs, LogToCoreLog(*log)) + logs = append(logs, ToCoreLog(*log)) } return logs } diff --git a/pkg/repositories/postgres.go b/pkg/repositories/postgres.go index 5461fe53..ef63d396 100644 --- a/pkg/repositories/postgres.go +++ b/pkg/repositories/postgres.go @@ -310,19 +310,30 @@ func (repository Postgres) createTransaction(tx *sql.Tx, blockId int64, transact if err != nil { return err } - if transaction.Receipt.TxHash != "" { - err = repository.createReceipt(tx, transactionId, transaction.Receipt) + if hasReceipt(transaction) { + receiptId, err := repository.createReceipt(tx, transactionId, transaction.Receipt) if err != nil { return err } - if len(transaction.Receipt.Logs) > 0 { - err = repository.CreateLogs(transaction.Receipt.Logs) + if hasLogs(transaction) { + err = repository.createLogs(tx, transaction.Receipt.Logs, receiptId) + if err != nil { + return err + } } } return nil } -func (repository Postgres) createReceipt(tx *sql.Tx, transactionId int, receipt core.Receipt) error { +func hasLogs(transaction core.Transaction) bool { + return len(transaction.Receipt.Logs) > 0 +} + +func hasReceipt(transaction core.Transaction) bool { + return transaction.Receipt.TxHash != "" +} + +func (repository Postgres) createReceipt(tx *sql.Tx, transactionId int, receipt core.Receipt) (int, error) { //Not currently persisting log bloom filters var receiptId int err := tx.QueryRow( @@ -332,7 +343,33 @@ func (repository Postgres) createReceipt(tx *sql.Tx, transactionId int, receipt RETURNING id`, receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, transactionId).Scan(&receiptId) if err != nil { - return err + return receiptId, err + } + return receiptId, nil +} + +func (repository Postgres) createLogs(tx *sql.Tx, logs []core.Log, receiptId int) error { + for _, tlog := range logs { + _, err := tx.Exec( + `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (index, block_number) + DO UPDATE + SET block_number = $1, + address = $2, + tx_hash = $3, + index = $4, + topic0 = $5, + topic1 = $6, + topic2 = $7, + topic3 = $8, + data = $9 + `, + tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, + ) + if err != nil { + return ErrDBInsertFailed + } } return nil } diff --git a/pkg/repositories/testing/helpers.go b/pkg/repositories/testing/helpers.go index 251d043d..6a89fd29 100644 --- a/pkg/repositories/testing/helpers.go +++ b/pkg/repositories/testing/helpers.go @@ -399,7 +399,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, Data: "xabc", }}, ) @@ -428,7 +428,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, Data: "xABC", }, }) @@ -437,7 +437,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, Data: "xXYZ", }, }) @@ -452,7 +452,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, Data: "xabc", }}, ) @@ -461,7 +461,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 1, Address: "x123", TxHash: "x789", - Topics: [4]string{0: "x111", 1: "x222", 2: "x333"}, + Topics: core.Topics{0: "x111", 1: "x222", 2: "x333"}, Data: "xdef", }}, ) @@ -470,7 +470,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, Data: "xabc", }}, ) @@ -561,13 +561,12 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Expect(err).To(Not(HaveOccurred())) retrievedLogs := repository.FindLogs("0x99041f808d598b782d5a3e498681c2452a31da08", 4745407) - Expect(retrievedLogs).To(Not(BeZero())) + expected := logs[1:] + Expect(retrievedLogs).To(Equal(expected)) }) - It("saves the logs attached to a receipt", func() { - logs := make([]core.Log, 0) + It("still saves receipts without logs", func() { receipt := core.Receipt{ - Logs: logs, TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", } transaction := core.Transaction{ @@ -580,11 +579,9 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. } repository.CreateOrUpdateBlock(block) - retrievedLogs := repository.FindLogs("0x99041f808d598b782d5a3e498681c2452a31da08", 4745407) _, err := repository.FindReceipt(receipt.TxHash) Expect(err).To(Not(HaveOccurred())) - Expect(retrievedLogs).To(BeZero()) }) }) From 08993cc6a4b43a2ccccc0397499fda7df0b98131 Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Tue, 16 Jan 2018 09:11:23 -0600 Subject: [PATCH 10/22] Add back schema dump --- Gododir/main.go | 4 + db/schema.sql | 476 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 480 insertions(+) create mode 100644 db/schema.sql diff --git a/Gododir/main.go b/Gododir/main.go index 09b49130..454e46e0 100644 --- a/Gododir/main.go +++ b/Gododir/main.go @@ -79,7 +79,9 @@ func tasks(p *do.Project) { cfg := cmd.LoadConfig(environment) connectString := config.DbConnectionString(cfg.Database) migrate := fmt.Sprintf("migrate -database '%s' -path ./db/migrations up", connectString) + dumpSchema := fmt.Sprintf("pg_dump -O -s %s > db/schema.sql", cfg.Database.Name) context.Bash(migrate) + context.Bash(dumpSchema) }) p.Task("rollback", nil, func(context *do.Context) { @@ -87,7 +89,9 @@ func tasks(p *do.Project) { cfg := cmd.LoadConfig(environment) connectString := config.DbConnectionString(cfg.Database) migrate := fmt.Sprintf("migrate -database '%s' -path ./db/migrations down 1", connectString) + dumpSchema := fmt.Sprintf("pg_dump -O -s %s > db/schema.sql", cfg.Database.Name) context.Bash(migrate) + context.Bash(dumpSchema) }) p.Task("showContractSummary", nil, func(context *do.Context) { diff --git a/db/schema.sql b/db/schema.sql new file mode 100644 index 00000000..0b7f6190 --- /dev/null +++ b/db/schema.sql @@ -0,0 +1,476 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 10.1 +-- Dumped by pg_dump version 10.1 + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; + + +-- +-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; + + +SET search_path = public, pg_catalog; + +SET default_tablespace = ''; + +SET default_with_oids = false; + +-- +-- Name: blocks; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE blocks ( + block_number bigint, + block_gaslimit double precision, + block_gasused double precision, + block_time double precision, + id integer NOT NULL, + block_difficulty bigint, + block_hash character varying(66), + block_nonce character varying(20), + block_parenthash character varying(66), + block_size bigint, + uncle_hash character varying(66), + node_id integer NOT NULL, + is_final boolean, + block_miner character varying(42), + block_extra_data character varying, + block_reward numeric, + block_uncles_reward numeric +); + + +-- +-- Name: blocks_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE blocks_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: blocks_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE blocks_id_seq OWNED BY blocks.id; + + +-- +-- Name: logs; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE logs ( + id integer NOT NULL, + block_number bigint, + address character varying(66), + tx_hash character varying(66), + index bigint, + topic0 character varying(66), + topic1 character varying(66), + topic2 character varying(66), + topic3 character varying(66), + data text, + receipt_id integer +); + + +-- +-- Name: logs_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE logs_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE logs_id_seq OWNED BY logs.id; + + +-- +-- Name: nodes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE nodes ( + id integer NOT NULL, + genesis_block character varying(66), + network_id numeric, + node_id character varying(128), + client_name character varying +); + + +-- +-- Name: nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE nodes_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: nodes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE nodes_id_seq OWNED BY nodes.id; + + +-- +-- Name: receipts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE receipts ( + id integer NOT NULL, + transaction_id integer NOT NULL, + contract_address character varying(42), + cumulative_gas_used numeric, + gas_used numeric, + state_root character varying(66), + status integer, + tx_hash character varying(66) +); + + +-- +-- Name: receipts_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE receipts_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: receipts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE receipts_id_seq OWNED BY receipts.id; + + +-- +-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE schema_migrations ( + version bigint NOT NULL, + dirty boolean NOT NULL +); + + +-- +-- Name: transactions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE transactions ( + id integer NOT NULL, + tx_hash character varying(66), + tx_nonce numeric, + tx_to character varying(66), + tx_gaslimit numeric, + tx_gasprice numeric, + tx_value numeric, + block_id integer NOT NULL, + tx_from character varying(66), + tx_input_data character varying +); + + +-- +-- Name: transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE transactions_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE transactions_id_seq OWNED BY transactions.id; + + +-- +-- Name: watched_contracts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE watched_contracts ( + contract_id integer NOT NULL, + contract_hash character varying(66), + contract_abi json +); + + +-- +-- Name: watched_contracts_contract_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE watched_contracts_contract_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: watched_contracts_contract_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE watched_contracts_contract_id_seq OWNED BY watched_contracts.contract_id; + + +-- +-- Name: blocks id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY blocks ALTER COLUMN id SET DEFAULT nextval('blocks_id_seq'::regclass); + + +-- +-- Name: logs id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY logs ALTER COLUMN id SET DEFAULT nextval('logs_id_seq'::regclass); + + +-- +-- Name: nodes id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY nodes ALTER COLUMN id SET DEFAULT nextval('nodes_id_seq'::regclass); + + +-- +-- Name: receipts id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY receipts ALTER COLUMN id SET DEFAULT nextval('receipts_id_seq'::regclass); + + +-- +-- Name: transactions id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY transactions ALTER COLUMN id SET DEFAULT nextval('transactions_id_seq'::regclass); + + +-- +-- Name: watched_contracts contract_id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY watched_contracts ALTER COLUMN contract_id SET DEFAULT nextval('watched_contracts_contract_id_seq'::regclass); + + +-- +-- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY blocks + ADD CONSTRAINT blocks_pkey PRIMARY KEY (id); + + +-- +-- Name: watched_contracts contract_hash_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY watched_contracts + ADD CONSTRAINT contract_hash_uc UNIQUE (contract_hash); + + +-- +-- Name: logs log_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY logs + ADD CONSTRAINT log_uc UNIQUE (block_number, index); + + +-- +-- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY logs + ADD CONSTRAINT logs_pkey PRIMARY KEY (id); + + +-- +-- Name: blocks node_id_block_number_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY blocks + ADD CONSTRAINT node_id_block_number_uc UNIQUE (block_number, node_id); + + +-- +-- Name: nodes node_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY nodes + ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id); + + +-- +-- Name: nodes nodes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY nodes + ADD CONSTRAINT nodes_pkey PRIMARY KEY (id); + + +-- +-- Name: receipts receipts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY receipts + ADD CONSTRAINT receipts_pkey PRIMARY KEY (id); + + +-- +-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY schema_migrations + ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); + + +-- +-- Name: transactions transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY transactions + ADD CONSTRAINT transactions_pkey PRIMARY KEY (id); + + +-- +-- Name: watched_contracts watched_contracts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY watched_contracts + ADD CONSTRAINT watched_contracts_pkey PRIMARY KEY (contract_id); + + +-- +-- Name: block_id_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX block_id_index ON transactions USING btree (block_id); + + +-- +-- Name: block_number_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX block_number_index ON blocks USING btree (block_number); + + +-- +-- Name: node_id_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX node_id_index ON blocks USING btree (node_id); + + +-- +-- Name: transaction_id_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX transaction_id_index ON receipts USING btree (transaction_id); + + +-- +-- Name: tx_from_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX tx_from_index ON transactions USING btree (tx_from); + + +-- +-- Name: tx_to_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX tx_to_index ON transactions USING btree (tx_to); + + +-- +-- Name: transactions blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY transactions + ADD CONSTRAINT blocks_fk FOREIGN KEY (block_id) REFERENCES blocks(id) ON DELETE CASCADE; + + +-- +-- Name: blocks node_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY blocks + ADD CONSTRAINT node_fk FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE; + + +-- +-- Name: logs receipts_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY logs + ADD CONSTRAINT receipts_fk FOREIGN KEY (receipt_id) REFERENCES receipts(id) ON DELETE CASCADE; + + +-- +-- Name: receipts transaction_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY receipts + ADD CONSTRAINT transaction_fk FOREIGN KEY (transaction_id) REFERENCES transactions(id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + From 6583ce72b82e3bea3d692a8855d102453ef8eddf Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Tue, 16 Jan 2018 09:28:19 -0600 Subject: [PATCH 11/22] Remove log_uc * Logs now are attached to receipt, so removing block + index unique constraint --- .../1516050071_add_log_fk_constraint.down.sql | 3 ++ .../1516050071_add_log_fk_constraint.up.sql | 2 ++ db/schema.sql | 8 ------ pkg/repositories/postgres.go | 28 ++----------------- pkg/repositories/testing/helpers.go | 24 ---------------- 5 files changed, 8 insertions(+), 57 deletions(-) diff --git a/db/migrations/1516050071_add_log_fk_constraint.down.sql b/db/migrations/1516050071_add_log_fk_constraint.down.sql index 6ce8446f..55779b51 100644 --- a/db/migrations/1516050071_add_log_fk_constraint.down.sql +++ b/db/migrations/1516050071_add_log_fk_constraint.down.sql @@ -6,4 +6,7 @@ ALTER TABLE logs ALTER TABLE logs DROP COLUMN receipt_id; +ALTER TABLE logs + ADD CONSTRAINT log_uc UNIQUE (block_number, index); + COMMIT; \ No newline at end of file diff --git a/db/migrations/1516050071_add_log_fk_constraint.up.sql b/db/migrations/1516050071_add_log_fk_constraint.up.sql index a0150d1a..60733b71 100644 --- a/db/migrations/1516050071_add_log_fk_constraint.up.sql +++ b/db/migrations/1516050071_add_log_fk_constraint.up.sql @@ -1,4 +1,6 @@ BEGIN; +ALTER TABLE logs + DROP CONSTRAINT log_uc; ALTER TABLE logs ADD COLUMN receipt_id INT; diff --git a/db/schema.sql b/db/schema.sql index 0b7f6190..e2e1ece3 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -324,14 +324,6 @@ ALTER TABLE ONLY watched_contracts ADD CONSTRAINT contract_hash_uc UNIQUE (contract_hash); --- --- Name: logs log_uc; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY logs - ADD CONSTRAINT log_uc UNIQUE (block_number, index); - - -- -- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- diff --git a/pkg/repositories/postgres.go b/pkg/repositories/postgres.go index ef63d396..14159547 100644 --- a/pkg/repositories/postgres.go +++ b/pkg/repositories/postgres.go @@ -351,21 +351,10 @@ func (repository Postgres) createReceipt(tx *sql.Tx, transactionId int, receipt func (repository Postgres) createLogs(tx *sql.Tx, logs []core.Log, receiptId int) error { for _, tlog := range logs { _, err := tx.Exec( - `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ON CONFLICT (index, block_number) - DO UPDATE - SET block_number = $1, - address = $2, - tx_hash = $3, - index = $4, - topic0 = $5, - topic1 = $6, - topic2 = $7, - topic3 = $8, - data = $9 + `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) `, - tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, + tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, receiptId, ) if err != nil { return ErrDBInsertFailed @@ -380,17 +369,6 @@ func (repository Postgres) CreateLogs(logs []core.Log) error { _, err := tx.Exec( `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ON CONFLICT (index, block_number) - DO UPDATE - SET block_number = $1, - address = $2, - tx_hash = $3, - index = $4, - topic0 = $5, - topic1 = $6, - topic2 = $7, - topic3 = $8, - data = $9 `, tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, ) diff --git a/pkg/repositories/testing/helpers.go b/pkg/repositories/testing/helpers.go index 6a89fd29..899b0987 100644 --- a/pkg/repositories/testing/helpers.go +++ b/pkg/repositories/testing/helpers.go @@ -422,30 +422,6 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Expect(log).To(BeNil()) }) - It("updates the log when log with when log with same block number and index is already present", func() { - repository.CreateLogs([]core.Log{{ - BlockNumber: 1, - Index: 0, - Address: "x123", - TxHash: "x456", - Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, - Data: "xABC", - }, - }) - repository.CreateLogs([]core.Log{{ - BlockNumber: 1, - Index: 0, - Address: "x123", - TxHash: "x456", - Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, - Data: "xXYZ", - }, - }) - - log := repository.FindLogs("x123", 1) - Expect(log[0].Data).To(Equal("xXYZ")) - }) - It("filters to the correct block number and address", func() { repository.CreateLogs([]core.Log{{ BlockNumber: 1, From 0ae25aff95cd46a58a3e1b17c1707e2278f4a5c7 Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Tue, 16 Jan 2018 09:48:42 -0600 Subject: [PATCH 12/22] Remove getLogs entrypoint --- Gododir/main.go | 14 --------- README.md | 6 ---- cmd/get_logs/main.go | 74 -------------------------------------------- 3 files changed, 94 deletions(-) delete mode 100644 cmd/get_logs/main.go diff --git a/Gododir/main.go b/Gododir/main.go index 454e46e0..6243f67f 100644 --- a/Gododir/main.go +++ b/Gododir/main.go @@ -42,20 +42,6 @@ func tasks(p *do.Project) { do.M{"environment": environment, "startingNumber": startingNumber, "$in": "cmd/populate_blocks"}) }) - p.Task("getLogs", nil, func(context *do.Context) { - environment := parseEnvironment(context) - contractHash := context.Args.MayString("", "contract-hash", "c") - if contractHash == "" { - log.Fatalln("--contract-hash required") - } - context.Start(`go run main.go --environment={{.environment}} --contract-hash={{.contractHash}}`, - do.M{ - "environment": environment, - "contractHash": contractHash, - "$in": "cmd/get_logs", - }) - }) - p.Task("watchContract", nil, func(context *do.Context) { environment := parseEnvironment(context) contractHash := context.Args.MayString("", "contract-hash", "c") diff --git a/README.md b/README.md index 832c00ab..e05adda6 100644 --- a/README.md +++ b/README.md @@ -91,12 +91,6 @@ The name of the JSON file should correspond the contract's address. 2. Start watching the contract `godo watchContract -- --environment= --contract-hash=` 3. Request summary data `godo showContractSummary -- --environment= --contract-hash=` - -## Retrieving Contract Logs - -1. Get the logs for a specific contract - - `godo getLogs -- --environment= --contract-hash=` - ### Configuring Additional Environments You can create configuration files for additional environments. diff --git a/cmd/get_logs/main.go b/cmd/get_logs/main.go deleted file mode 100644 index 3e013882..00000000 --- a/cmd/get_logs/main.go +++ /dev/null @@ -1,74 +0,0 @@ -package main - -import ( - "log" - - "flag" - - "math/big" - - "time" - - "strings" - - "github.com/8thlight/vulcanizedb/cmd" - "github.com/8thlight/vulcanizedb/pkg/core" - "github.com/8thlight/vulcanizedb/pkg/geth" -) - -func min(a, b int64) int64 { - if a < b { - return a - } - return b -} - -const ( - windowSize = 24 - pollingInterval = 10 * time.Second -) - -func main() { - environment := flag.String("environment", "", "Environment name") - contractHash := flag.String("contract-hash", "", "Contract hash to show summary") - flag.Parse() - - ticker := time.NewTicker(pollingInterval) - defer ticker.Stop() - - contractHashLowered := strings.ToLower(*contractHash) - config := cmd.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) - - lastBlockNumber := blockchain.LastBlock().Int64() - stepSize := int64(1000) - - go func() { - for i := int64(0); i < lastBlockNumber; i = min(i+stepSize, lastBlockNumber) { - logs, err := blockchain.GetLogs(core.Contract{Hash: contractHashLowered}, big.NewInt(i), big.NewInt(i+stepSize)) - log.Println("Backfilling Logs:", i) - if err != nil { - log.Println(err) - } - repository.CreateLogs(logs) - } - }() - - done := make(chan struct{}) - go func() { done <- struct{}{} }() - for range ticker.C { - select { - case <-done: - go func() { - z := &big.Int{} - z.Sub(blockchain.LastBlock(), big.NewInt(25)) - log.Printf("Logs Window: %d - %d", z.Int64(), blockchain.LastBlock().Int64()) - logs, _ := blockchain.GetLogs(core.Contract{Hash: contractHashLowered}, z, blockchain.LastBlock()) - repository.CreateLogs(logs) - done <- struct{}{} - }() - default: - } - } -} From 3f06c7374baafec5222745e22df437e3cd2d1945 Mon Sep 17 00:00:00 2001 From: Matt K <1036969+mkrump@users.noreply.github.com> Date: Tue, 16 Jan 2018 14:25:33 -0600 Subject: [PATCH 13/22] Update transaction value field (#132) * Add test for converting big.Int * Use string as internal representation of transaction value --- pkg/core/transaction.go | 2 +- pkg/geth/block_to_core_block.go | 2 +- pkg/geth/block_to_core_block_test.go | 2 +- pkg/repositories/postgres.go | 4 +-- pkg/repositories/postgres_test.go | 38 ++++++++++++++++++++++++++++ pkg/repositories/testing/helpers.go | 9 ++++--- 6 files changed, 49 insertions(+), 8 deletions(-) diff --git a/pkg/core/transaction.go b/pkg/core/transaction.go index 0aa900ab..585b4935 100644 --- a/pkg/core/transaction.go +++ b/pkg/core/transaction.go @@ -9,5 +9,5 @@ type Transaction struct { GasLimit int64 GasPrice int64 Receipt - Value int64 + Value string } diff --git a/pkg/geth/block_to_core_block.go b/pkg/geth/block_to_core_block.go index 8cc806e8..ebed70ca 100644 --- a/pkg/geth/block_to_core_block.go +++ b/pkg/geth/block_to_core_block.go @@ -76,7 +76,7 @@ func transToCoreTrans(transaction *types.Transaction, from *common.Address) core From: strings.ToLower(addressToHex(from)), GasLimit: transaction.Gas().Int64(), GasPrice: transaction.GasPrice().Int64(), - Value: transaction.Value().Int64(), + Value: transaction.Value().String(), Data: data, } } diff --git a/pkg/geth/block_to_core_block_test.go b/pkg/geth/block_to_core_block_test.go index 22c0276b..6dc03ec5 100644 --- a/pkg/geth/block_to_core_block_test.go +++ b/pkg/geth/block_to_core_block_test.go @@ -246,7 +246,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() { Expect(coreTransaction.From).To(Equal("0x0000000000000000000000000000000000000123")) Expect(coreTransaction.GasLimit).To(Equal(gethTransaction.Gas().Int64())) Expect(coreTransaction.GasPrice).To(Equal(gethTransaction.GasPrice().Int64())) - Expect(coreTransaction.Value).To(Equal(gethTransaction.Value().Int64())) + Expect(coreTransaction.Value).To(Equal(gethTransaction.Value().String())) Expect(coreTransaction.Nonce).To(Equal(gethTransaction.Nonce())) coreReceipt := coreTransaction.Receipt diff --git a/pkg/repositories/postgres.go b/pkg/repositories/postgres.go index 14159547..fdc92f74 100644 --- a/pkg/repositories/postgres.go +++ b/pkg/repositories/postgres.go @@ -303,7 +303,7 @@ func (repository Postgres) createTransaction(tx *sql.Tx, blockId int64, transact err := tx.QueryRow( `INSERT INTO transactions (block_id, tx_hash, tx_nonce, tx_to, tx_from, tx_gaslimit, tx_gasprice, tx_value, tx_input_data) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + VALUES ($1, $2, $3, $4, $5, $6, $7, cast(NULLIF($8, '') as NUMERIC), $9) RETURNING id`, blockId, transaction.Hash, transaction.Nonce, transaction.To, transaction.From, transaction.GasLimit, transaction.GasPrice, transaction.Value, transaction.Data). Scan(&transactionId) @@ -489,7 +489,7 @@ func (repository Postgres) loadTransactions(transactionRows *sql.Rows) []core.Tr var gasLimit int64 var gasPrice int64 var inputData string - var value int64 + var value string transactionRows.Scan(&hash, &nonce, &to, &from, &gasLimit, &gasPrice, &value, &inputData) transaction := core.Transaction{ Hash: hash, diff --git a/pkg/repositories/postgres_test.go b/pkg/repositories/postgres_test.go index ce75c2c5..3e19abfc 100644 --- a/pkg/repositories/postgres_test.go +++ b/pkg/repositories/postgres_test.go @@ -7,6 +7,8 @@ import ( "io/ioutil" "log" + "math/big" + "github.com/8thlight/vulcanizedb/pkg/config" "github.com/8thlight/vulcanizedb/pkg/core" "github.com/8thlight/vulcanizedb/pkg/repositories" @@ -38,6 +40,42 @@ var _ = Describe("Postgres repository", func() { return repository }) + It("serializes big.Int to db", func() { + // postgres driver doesn't support go big.Int type + // various casts in golang uint64, int64, overflow for + // transaction value (in wei) even though + // postgres numeric can handle an arbitrary + // sized int, so use string representation of big.Int + // and cast on insert + + cfg, _ := config.NewConfig("private") + pgConfig := config.DbConnectionString(cfg.Database) + db, err := sqlx.Connect("postgres", pgConfig) + + bi := new(big.Int) + bi.SetString("34940183920000000000", 10) + Expect(bi.String()).To(Equal("34940183920000000000")) + + defer db.Exec(`DROP TABLE IF EXISTS example`) + _, err = db.Exec("CREATE TABLE example ( id INTEGER, data NUMERIC )") + Expect(err).ToNot(HaveOccurred()) + + sqlStatement := ` + INSERT INTO example (id, data) + VALUES (1, cast($1 AS NUMERIC))` + _, err = db.Exec(sqlStatement, bi.String()) + Expect(err).ToNot(HaveOccurred()) + + var data string + err = db.QueryRow(`SELECT data FROM example WHERE id = 1`).Scan(&data) + Expect(err).ToNot(HaveOccurred()) + + Expect(bi.String()).To(Equal(data)) + actual := new(big.Int) + actual.SetString(data, 10) + Expect(actual).To(Equal(bi)) + }) + It("does not commit block if block is invalid", func() { //badNonce violates db Nonce field length badNonce := fmt.Sprintf("x %s", strings.Repeat("1", 100)) diff --git a/pkg/repositories/testing/helpers.go b/pkg/repositories/testing/helpers.go index 899b0987..c258f041 100644 --- a/pkg/repositories/testing/helpers.go +++ b/pkg/repositories/testing/helpers.go @@ -4,6 +4,8 @@ import ( "sort" "strconv" + "math/big" + "github.com/8thlight/vulcanizedb/pkg/core" "github.com/8thlight/vulcanizedb/pkg/repositories" . "github.com/onsi/ginkgo" @@ -198,7 +200,8 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. nonce := uint64(10000) to := "1234567890" from := "0987654321" - value := int64(10) + var value = new(big.Int) + value.SetString("34940183920000000000", 10) inputData := "0xf7d8c8830000000000000000000000000000000000000000000000000000000000037788000000000000000000000000000000000000000000000000000000000003bd14" transaction := core.Transaction{ Hash: "x1234", @@ -207,7 +210,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Nonce: nonce, To: to, From: from, - Value: value, + Value: value.String(), Data: inputData, } block := core.Block{ @@ -227,7 +230,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Expect(savedTransaction.Nonce).To(Equal(nonce)) Expect(savedTransaction.GasLimit).To(Equal(gasLimit)) Expect(savedTransaction.GasPrice).To(Equal(gasPrice)) - Expect(savedTransaction.Value).To(Equal(value)) + Expect(savedTransaction.Value).To(Equal(value.String())) }) }) From c00b8a5a9805490e8ec0fd50ef3a887b270085c8 Mon Sep 17 00:00:00 2001 From: Matt K <1036969+mkrump@users.noreply.github.com> Date: Tue, 23 Jan 2018 12:43:35 -0600 Subject: [PATCH 14/22] Add Filters (#133) * Add LogFilter struct * Add log_filters table * Add view for events watching * Add cmd line "add_filter" to mimic eventual endpoint * Allow multiple filters in config --- .gitignore | 1 + Gododir/main.go | 23 +++- README.md | 13 +- cmd/add_filter/main.go | 38 ++++++ cmd/utils.go | 11 +- cmd/vulcanize_db/main.go | 10 +- .../1516648743_add_log_filters.down.sql | 1 + .../1516648743_add_log_filters.up.sql | 12 ++ ...1516653373_add_watched_event_logs.down.sql | 1 + .../1516653373_add_watched_event_logs.up.sql | 29 ++++ db/schema.sql | 102 +++++++++++++- filters/example-filter.json | 15 +++ pkg/filters/filter_query.go | 64 +++++++++ pkg/filters/filter_test.go | 125 ++++++++++++++++++ pkg/filters/query_builder_suite_test.go | 13 ++ pkg/repositories/in_memory.go | 22 ++- pkg/repositories/postgres.go | 15 ++- pkg/repositories/repository.go | 6 +- pkg/repositories/testing/helpers.go | 39 ++++++ 19 files changed, 514 insertions(+), 26 deletions(-) create mode 100644 cmd/add_filter/main.go create mode 100644 db/migrations/1516648743_add_log_filters.down.sql create mode 100644 db/migrations/1516648743_add_log_filters.up.sql create mode 100644 db/migrations/1516653373_add_watched_event_logs.down.sql create mode 100644 db/migrations/1516653373_add_watched_event_logs.up.sql create mode 100644 filters/example-filter.json create mode 100644 pkg/filters/filter_query.go create mode 100644 pkg/filters/filter_test.go create mode 100644 pkg/filters/query_builder_suite_test.go diff --git a/.gitignore b/.gitignore index ac6ee651..f5b5e5ae 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ environments/*.toml Vagrantfile vagrant_bootstrap.sh .vagrant +test_scripts/ diff --git a/Gododir/main.go b/Gododir/main.go index 6243f67f..15200298 100644 --- a/Gododir/main.go +++ b/Gododir/main.go @@ -28,13 +28,16 @@ func tasks(p *do.Project) { p.Task("vulcanizeDb", nil, func(context *do.Context) { environment := parseEnvironment(context) - context.Start(`go run main.go --environment={{.environment}}`, - do.M{"environment": environment, "$in": "cmd/vulcanize_db"}) + startingNumber := context.Args.MayInt(0, "starting-number", "s") + context.Start(`go run main.go --environment={{.environment}} --starting-number={{.startingNumber}}`, + do.M{"environment": environment, + "startingNumber": startingNumber, + "$in": "cmd/vulcanize_db"}) }) p.Task("populateBlocks", nil, func(context *do.Context) { environment := parseEnvironment(context) - startingNumber := context.Args.MayInt(-1, "starting-number") + startingNumber := context.Args.MayInt(-1, "starting-number", "s") if startingNumber < 0 { log.Fatalln("--starting-number required") } @@ -42,6 +45,20 @@ func tasks(p *do.Project) { do.M{"environment": environment, "startingNumber": startingNumber, "$in": "cmd/populate_blocks"}) }) + p.Task("watchEvent", nil, func(context *do.Context) { + environment := parseEnvironment(context) + filterFilePath := context.Args.MayString("", "filter-filepath", "f") + if filterFilePath == "" { + log.Fatalln("--filter-filepath required") + } + context.Start(`go run main.go --environment={{.environment}} --filter-filepath={{.filterFilePath}}`, + do.M{ + "environment": environment, + "filterFilePath": filterFilePath, + "$in": "cmd/add_filter", + }) + }) + p.Task("watchContract", nil, func(context *do.Context) { environment := parseEnvironment(context) contractHash := context.Args.MayString("", "contract-hash", "c") diff --git a/README.md b/README.md index e05adda6..b8dc09e3 100644 --- a/README.md +++ b/README.md @@ -60,19 +60,26 @@ The default location for Ethereum is: **Note the location of the ipc file is outputted when you connect to a blockchain. It is needed to for configuration** ## Start Vulcanize DB -1. Start a blockchain. +1. Start geth 2. In a separate terminal start vulcanize_db - `godo vulcanizeDb -- --environment=` +## Watch contract events +1. Start geth +2. In a separate terminal start vulcanize_db + - `godo vulcanizeDb -- --environment=` +3. Create event filter + - `godo watchEvent -- --environment= --filter-filepath=` + ## Running Listener -1. Start a blockchain. +1. Start geth 2. In a separate terminal start listener (ipcDir location) - `godo run -- --environment=` ## Retrieving Historical Data -1. Start a blockchain. +1. Start geth 2. In a separate terminal start listener (ipcDir location) - `godo populateBlocks -- --environment= --starting-number=` diff --git a/cmd/add_filter/main.go b/cmd/add_filter/main.go new file mode 100644 index 00000000..026c0854 --- /dev/null +++ b/cmd/add_filter/main.go @@ -0,0 +1,38 @@ +package main + +import ( + "encoding/json" + "flag" + "io/ioutil" + "log" + + "github.com/8thlight/vulcanizedb/cmd" + "github.com/8thlight/vulcanizedb/pkg/filters" + "github.com/8thlight/vulcanizedb/pkg/geth" +) + +func main() { + environment := flag.String("environment", "", "Environment name") + filterFilePath := flag.String("filter-filepath", "", "path/to/filter.json") + + flag.Parse() + var logFilters filters.LogFilters + config := cmd.LoadConfig(*environment) + blockchain := geth.NewBlockchain(config.Client.IPCPath) + repository := cmd.LoadPostgres(config.Database, blockchain.Node()) + absFilePath := cmd.AbsFilePath(*filterFilePath) + logFilterBytes, err := ioutil.ReadFile(absFilePath) + if err != nil { + log.Fatal(err) + } + err = json.Unmarshal(logFilterBytes, &logFilters) + if err != nil { + log.Fatal(err) + } + for _, filter := range logFilters { + err = repository.AddFilter(filter) + if err != nil { + log.Fatal(err) + } + } +} diff --git a/cmd/utils.go b/cmd/utils.go index c7e5f8db..03a6be65 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -30,9 +30,7 @@ func LoadPostgres(database config.Database, node core.Node) repositories.Postgre } func ReadAbiFile(abiFilepath string) string { - if !filepath.IsAbs(abiFilepath) { - abiFilepath = filepath.Join(config.ProjectRoot(), abiFilepath) - } + abiFilepath = AbsFilePath(abiFilepath) abi, err := geth.ReadAbiFile(abiFilepath) if err != nil { log.Fatalf("Error reading ABI file at \"%s\"\n %v", abiFilepath, err) @@ -40,6 +38,13 @@ func ReadAbiFile(abiFilepath string) string { return abi } +func AbsFilePath(filePath string) string { + if !filepath.IsAbs(filePath) { + filePath = filepath.Join(config.ProjectRoot(), filePath) + } + return filePath +} + func GetAbi(abiFilepath string, contractHash string, network string) string { var contractAbiString string if abiFilepath != "" { diff --git a/cmd/vulcanize_db/main.go b/cmd/vulcanize_db/main.go index b74be6b5..24cf0776 100644 --- a/cmd/vulcanize_db/main.go +++ b/cmd/vulcanize_db/main.go @@ -18,14 +18,15 @@ const ( pollingInterval = 7 * time.Second ) -func backFillAllBlocks(blockchain core.Blockchain, repository repositories.Postgres, missingBlocksPopulated chan int) { +func backFillAllBlocks(blockchain core.Blockchain, repository repositories.Postgres, missingBlocksPopulated chan int, startingBlockNumber int64) { go func() { - missingBlocksPopulated <- history.PopulateMissingBlocks(blockchain, repository, 0) + missingBlocksPopulated <- history.PopulateMissingBlocks(blockchain, repository, startingBlockNumber) }() } func main() { environment := flag.String("environment", "", "Environment name") + startingBlockNumber := flag.Int("starting-number", 0, "First block to fill from") flag.Parse() ticker := time.NewTicker(pollingInterval) @@ -37,7 +38,8 @@ func main() { validator := history.NewBlockValidator(blockchain, repository, 15) missingBlocksPopulated := make(chan int) - go backFillAllBlocks(blockchain, repository, missingBlocksPopulated) + _startingBlockNumber := int64(*startingBlockNumber) + go backFillAllBlocks(blockchain, repository, missingBlocksPopulated, _startingBlockNumber) for { select { @@ -45,7 +47,7 @@ func main() { window := validator.ValidateBlocks() validator.Log(os.Stdout, window) case <-missingBlocksPopulated: - go backFillAllBlocks(blockchain, repository, missingBlocksPopulated) + go backFillAllBlocks(blockchain, repository, missingBlocksPopulated, _startingBlockNumber) } } } diff --git a/db/migrations/1516648743_add_log_filters.down.sql b/db/migrations/1516648743_add_log_filters.down.sql new file mode 100644 index 00000000..a28adeee --- /dev/null +++ b/db/migrations/1516648743_add_log_filters.down.sql @@ -0,0 +1 @@ +DROP TABLE log_filters; \ No newline at end of file diff --git a/db/migrations/1516648743_add_log_filters.up.sql b/db/migrations/1516648743_add_log_filters.up.sql new file mode 100644 index 00000000..100a5718 --- /dev/null +++ b/db/migrations/1516648743_add_log_filters.up.sql @@ -0,0 +1,12 @@ +CREATE TABLE log_filters ( + id SERIAL, + name VARCHAR NOT NULL CHECK (name <> ''), + from_block BIGINT CHECK (from_block >= 0), + to_block BIGINT CHECK (from_block >= 0), + address VARCHAR(66), + topic0 VARCHAR(66), + topic1 VARCHAR(66), + topic2 VARCHAR(66), + topic3 VARCHAR(66), + CONSTRAINT name_uc UNIQUE (name) +); \ No newline at end of file diff --git a/db/migrations/1516653373_add_watched_event_logs.down.sql b/db/migrations/1516653373_add_watched_event_logs.down.sql new file mode 100644 index 00000000..14bb2987 --- /dev/null +++ b/db/migrations/1516653373_add_watched_event_logs.down.sql @@ -0,0 +1 @@ +DROP VIEW watched_event_logs; \ No newline at end of file diff --git a/db/migrations/1516653373_add_watched_event_logs.up.sql b/db/migrations/1516653373_add_watched_event_logs.up.sql new file mode 100644 index 00000000..3bba916f --- /dev/null +++ b/db/migrations/1516653373_add_watched_event_logs.up.sql @@ -0,0 +1,29 @@ +CREATE VIEW block_stats AS + SELECT + max(block_number) AS max_block, + min(block_number) AS min_block + FROM logs; + +CREATE VIEW watched_event_logs AS + SELECT + log_filters.name, + logs.id, + block_number, + logs.address, + tx_hash, + index, + logs.topic0, + logs.topic1, + logs.topic2, + logs.topic3, + data, + receipt_id + FROM log_filters + CROSS JOIN block_stats + JOIN logs ON logs.address = log_filters.address + AND logs.block_number >= coalesce(log_filters.from_block, block_stats.min_block) + AND logs.block_number <= coalesce(log_filters.to_block, block_stats.max_block) + WHERE (log_filters.topic0 = logs.topic0 OR log_filters.topic0 ISNULL) + AND (log_filters.topic1 = logs.topic1 OR log_filters.topic1 ISNULL) + AND (log_filters.topic2 = logs.topic2 OR log_filters.topic2 ISNULL) + AND (log_filters.topic3 = logs.topic3 OR log_filters.topic3 ISNULL); \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index e2e1ece3..9ba46900 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -34,6 +34,35 @@ SET default_tablespace = ''; SET default_with_oids = false; +-- +-- Name: logs; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE logs ( + id integer NOT NULL, + block_number bigint, + address character varying(66), + tx_hash character varying(66), + index bigint, + topic0 character varying(66), + topic1 character varying(66), + topic2 character varying(66), + topic3 character varying(66), + data text, + receipt_id integer +); + + +-- +-- Name: block_stats; Type: VIEW; Schema: public; Owner: - +-- + +CREATE VIEW block_stats AS + SELECT max(logs.block_number) AS max_block, + min(logs.block_number) AS min_block + FROM logs; + + -- -- Name: blocks; Type: TABLE; Schema: public; Owner: - -- @@ -80,24 +109,45 @@ ALTER SEQUENCE blocks_id_seq OWNED BY blocks.id; -- --- Name: logs; Type: TABLE; Schema: public; Owner: - +-- Name: log_filters; Type: TABLE; Schema: public; Owner: - -- -CREATE TABLE logs ( +CREATE TABLE log_filters ( id integer NOT NULL, - block_number bigint, + name character varying NOT NULL, + from_block bigint, + to_block bigint, address character varying(66), - tx_hash character varying(66), - index bigint, topic0 character varying(66), topic1 character varying(66), topic2 character varying(66), topic3 character varying(66), - data text, - receipt_id integer + CONSTRAINT log_filters_from_block_check CHECK ((from_block >= 0)), + CONSTRAINT log_filters_from_block_check1 CHECK ((from_block >= 0)), + CONSTRAINT log_filters_name_check CHECK (((name)::text <> ''::text)) ); +-- +-- Name: log_filters_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE log_filters_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: log_filters_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE log_filters_id_seq OWNED BY log_filters.id; + + -- -- Name: logs_id_seq; Type: SEQUENCE; Schema: public; Owner: - -- @@ -266,6 +316,29 @@ CREATE SEQUENCE watched_contracts_contract_id_seq ALTER SEQUENCE watched_contracts_contract_id_seq OWNED BY watched_contracts.contract_id; +-- +-- Name: watched_event_logs; Type: VIEW; Schema: public; Owner: - +-- + +CREATE VIEW watched_event_logs AS + SELECT log_filters.name, + logs.id, + logs.block_number, + logs.address, + logs.tx_hash, + logs.index, + logs.topic0, + logs.topic1, + logs.topic2, + logs.topic3, + logs.data, + logs.receipt_id + FROM ((log_filters + CROSS JOIN block_stats) + JOIN logs ON ((((logs.address)::text = (log_filters.address)::text) AND (logs.block_number >= COALESCE(log_filters.from_block, block_stats.min_block)) AND (logs.block_number <= COALESCE(log_filters.to_block, block_stats.max_block))))) + WHERE ((((log_filters.topic0)::text = (logs.topic0)::text) OR (log_filters.topic0 IS NULL)) AND (((log_filters.topic1)::text = (logs.topic1)::text) OR (log_filters.topic1 IS NULL)) AND (((log_filters.topic2)::text = (logs.topic2)::text) OR (log_filters.topic2 IS NULL)) AND (((log_filters.topic3)::text = (logs.topic3)::text) OR (log_filters.topic3 IS NULL))); + + -- -- Name: blocks id; Type: DEFAULT; Schema: public; Owner: - -- @@ -273,6 +346,13 @@ ALTER SEQUENCE watched_contracts_contract_id_seq OWNED BY watched_contracts.cont ALTER TABLE ONLY blocks ALTER COLUMN id SET DEFAULT nextval('blocks_id_seq'::regclass); +-- +-- Name: log_filters id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY log_filters ALTER COLUMN id SET DEFAULT nextval('log_filters_id_seq'::regclass); + + -- -- Name: logs id; Type: DEFAULT; Schema: public; Owner: - -- @@ -332,6 +412,14 @@ ALTER TABLE ONLY logs ADD CONSTRAINT logs_pkey PRIMARY KEY (id); +-- +-- Name: log_filters name_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY log_filters + ADD CONSTRAINT name_uc UNIQUE (name); + + -- -- Name: blocks node_id_block_number_uc; Type: CONSTRAINT; Schema: public; Owner: - -- diff --git a/filters/example-filter.json b/filters/example-filter.json new file mode 100644 index 00000000..b123b0eb --- /dev/null +++ b/filters/example-filter.json @@ -0,0 +1,15 @@ +[{ +"name": "TransferFilter", +"fromBlock": "0x488290", +"toBlock": "0x488678", +"address": "0x06012c8cf97bead5deae237070f9587f8e7a266d", +"topics": ["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"] +}, +{ + "name": "NewFilter", + "toBlock": "0x4B34AA", + "fromBlock": "0x4B34AD", + "address": "0x06012c8cf97bead5deae237070f9587f8e7a266d", + "topics": ["0x241ea03ca20251805084d27d4440371c34a0b85ff108f6bb5611248f73818b80"] +}] + diff --git a/pkg/filters/filter_query.go b/pkg/filters/filter_query.go new file mode 100644 index 00000000..0fb4e8bd --- /dev/null +++ b/pkg/filters/filter_query.go @@ -0,0 +1,64 @@ +package filters + +import ( + "encoding/json" + + "errors" + + "github.com/8thlight/vulcanizedb/pkg/core" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +type LogFilters []LogFilter + +type LogFilter struct { + Name string `json:"name"` + FromBlock int64 `json:"fromBlock"` + ToBlock int64 `json:"toBlock"` + Address string `json:"address"` + core.Topics `json:"topics"` +} + +func (filterQuery *LogFilter) UnmarshalJSON(input []byte) error { + type Alias LogFilter + + var err error + aux := &struct { + ToBlock string `json:"toBlock"` + FromBlock string `json:"fromBlock"` + *Alias + }{ + Alias: (*Alias)(filterQuery), + } + if err := json.Unmarshal(input, &aux); err != nil { + return err + } + if filterQuery.Name == "" { + return errors.New("filters: must provide name for logfilter") + } + filterQuery.ToBlock, err = filterQuery.unmarshalFromToBlock(aux.ToBlock) + if err != nil { + return errors.New("filters: invalid fromBlock") + } + filterQuery.FromBlock, err = filterQuery.unmarshalFromToBlock(aux.FromBlock) + if err != nil { + return errors.New("filters: invalid fromBlock") + } + if !common.IsHexAddress(filterQuery.Address) { + return errors.New("filters: invalid address") + } + + return nil +} + +func (filterQuery *LogFilter) unmarshalFromToBlock(auxBlock string) (int64, error) { + if auxBlock == "" { + return -1, nil + } + block, err := hexutil.DecodeUint64(auxBlock) + if err != nil { + return 0, errors.New("filters: invalid block arg") + } + return int64(block), nil +} diff --git a/pkg/filters/filter_test.go b/pkg/filters/filter_test.go new file mode 100644 index 00000000..c2978771 --- /dev/null +++ b/pkg/filters/filter_test.go @@ -0,0 +1,125 @@ +package filters_test + +import ( + "encoding/json" + + "github.com/8thlight/vulcanizedb/pkg/core" + "github.com/8thlight/vulcanizedb/pkg/filters" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Log filters", func() { + It("decodes web3 filter to LogFilter", func() { + + var logFilter filters.LogFilter + jsonFilter := []byte( + `{ + "name": "TestEvent", + "fromBlock": "0x1", + "toBlock": "0x488290", + "address": "0x8888f1f195afa192cfee860698584c030f4c9db1", + "topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null, "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null] + }`) + err := json.Unmarshal(jsonFilter, &logFilter) + + Expect(err).ToNot(HaveOccurred()) + Expect(logFilter.Name).To(Equal("TestEvent")) + Expect(logFilter.FromBlock).To(Equal(int64(1))) + Expect(logFilter.ToBlock).To(Equal(int64(4752016))) + Expect(logFilter.Address).To(Equal("0x8888f1f195afa192cfee860698584c030f4c9db1")) + Expect(logFilter.Topics).To(Equal( + core.Topics{ + "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "", + "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + ""})) + }) + + It("decodes array of web3 filters to []LogFilter", func() { + + logFilters := make([]filters.LogFilter, 0) + jsonFilter := []byte( + `[{ + "name": "TestEvent", + "fromBlock": "0x1", + "toBlock": "0x488290", + "address": "0x8888f1f195afa192cfee860698584c030f4c9db1", + "topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null, "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null] + }, + { + "name": "TestEvent2", + "fromBlock": "0x3", + "toBlock": "0x4", + "address": "0xd26114cd6EE289AccF82350c8d8487fedB8A0C07", + "topics": ["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", "0x0000000000000000000000006b0949d4c6edfe467db78241b7d5566f3c2bb43e", "0x0000000000000000000000005e44c3e467a49c9ca0296a9f130fc433041aaa28"] + }]`) + err := json.Unmarshal(jsonFilter, &logFilters) + + Expect(err).ToNot(HaveOccurred()) + Expect(len(logFilters)).To(Equal(2)) + Expect(logFilters[0].Name).To(Equal("TestEvent")) + Expect(logFilters[1].Name).To(Equal("TestEvent2")) + }) + + It("requires valid ethereum address", func() { + + var logFilter filters.LogFilter + jsonFilter := []byte( + `{ + "name": "TestEvent", + "fromBlock": "0x1", + "toBlock": "0x2", + "address": "0x8888f1f195afa192cf84c030f4c9db1", + "topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null, "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null] + }`) + err := json.Unmarshal(jsonFilter, &logFilter) + Expect(err).To(HaveOccurred()) + + }) + It("requires name", func() { + + var logFilter filters.LogFilter + jsonFilter := []byte( + `{ + "fromBlock": "0x1", + "toBlock": "0x2", + "address": "0x8888f1f195afa192cfee860698584c030f4c9db1", + "topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null, "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null] + }`) + err := json.Unmarshal(jsonFilter, &logFilter) + Expect(err).To(HaveOccurred()) + + }) + + It("maps missing fromBlock to -1", func() { + + var logFilter filters.LogFilter + jsonFilter := []byte( + `{ + "name": "TestEvent", + "toBlock": "0x2", + "address": "0x8888f1f195afa192cfee860698584c030f4c9db1", + "topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null, "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null] + }`) + err := json.Unmarshal(jsonFilter, &logFilter) + Expect(err).ToNot(HaveOccurred()) + Expect(logFilter.FromBlock).To(Equal(int64(-1))) + + }) + + It("maps missing toBlock to -1", func() { + var logFilter filters.LogFilter + jsonFilter := []byte( + `{ + "name": "TestEvent", + "address": "0x8888f1f195afa192cfee860698584c030f4c9db1", + "topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null, "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null] + }`) + err := json.Unmarshal(jsonFilter, &logFilter) + Expect(err).ToNot(HaveOccurred()) + Expect(logFilter.ToBlock).To(Equal(int64(-1))) + + }) + +}) diff --git a/pkg/filters/query_builder_suite_test.go b/pkg/filters/query_builder_suite_test.go new file mode 100644 index 00000000..42029812 --- /dev/null +++ b/pkg/filters/query_builder_suite_test.go @@ -0,0 +1,13 @@ +package filters_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestQueryBuilder(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "QueryBuilder Suite") +} diff --git a/pkg/repositories/in_memory.go b/pkg/repositories/in_memory.go index 0a609231..838cdefc 100644 --- a/pkg/repositories/in_memory.go +++ b/pkg/repositories/in_memory.go @@ -3,7 +3,10 @@ package repositories import ( "fmt" + "errors" + "github.com/8thlight/vulcanizedb/pkg/core" + "github.com/8thlight/vulcanizedb/pkg/filters" ) type InMemory struct { @@ -11,16 +14,27 @@ type InMemory struct { receipts map[string]core.Receipt contracts map[string]core.Contract logs map[string][]core.Log + logFilters map[string]filters.LogFilter CreateOrUpdateBlockCallCount int } +func (repository *InMemory) AddFilter(filter filters.LogFilter) error { + key := filter.Name + if _, ok := repository.logFilters[key]; ok || key == "" { + return errors.New("filter name not unique") + } + repository.logFilters[key] = filter + return nil +} + func NewInMemory() *InMemory { return &InMemory{ CreateOrUpdateBlockCallCount: 0, - blocks: make(map[int64]core.Block), - receipts: make(map[string]core.Receipt), - contracts: make(map[string]core.Contract), - logs: make(map[string][]core.Log), + blocks: make(map[int64]core.Block), + receipts: make(map[string]core.Receipt), + contracts: make(map[string]core.Contract), + logs: make(map[string][]core.Log), + logFilters: make(map[string]filters.LogFilter), } } diff --git a/pkg/repositories/postgres.go b/pkg/repositories/postgres.go index fdc92f74..11fa5f23 100644 --- a/pkg/repositories/postgres.go +++ b/pkg/repositories/postgres.go @@ -11,6 +11,7 @@ import ( "github.com/8thlight/vulcanizedb/pkg/config" "github.com/8thlight/vulcanizedb/pkg/core" + "github.com/8thlight/vulcanizedb/pkg/filters" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" ) @@ -303,7 +304,7 @@ func (repository Postgres) createTransaction(tx *sql.Tx, blockId int64, transact err := tx.QueryRow( `INSERT INTO transactions (block_id, tx_hash, tx_nonce, tx_to, tx_from, tx_gaslimit, tx_gasprice, tx_value, tx_input_data) - VALUES ($1, $2, $3, $4, $5, $6, $7, cast(NULLIF($8, '') as NUMERIC), $9) + VALUES ($1, $2, $3, $4, $5, $6, $7, cast(NULLIF($8, '') AS NUMERIC), $9) RETURNING id`, blockId, transaction.Hash, transaction.Nonce, transaction.To, transaction.From, transaction.GasLimit, transaction.GasPrice, transaction.Value, transaction.Data). Scan(&transactionId) @@ -381,6 +382,18 @@ func (repository Postgres) CreateLogs(logs []core.Log) error { return nil } +func (repository Postgres) AddFilter(query filters.LogFilter) error { + _, err := repository.Db.Exec( + `INSERT INTO log_filters + (name, from_block, to_block, address, topic0, topic1, topic2, topic3) + VALUES ($1, NULLIF($2, -1), NULLIF($3, -1), $4, NULLIF($5, ''), NULLIF($6, ''), NULLIF($7, ''), NULLIF($8, ''))`, + query.Name, query.FromBlock, query.ToBlock, query.Address, query.Topics[0], query.Topics[1], query.Topics[2], query.Topics[3]) + if err != nil { + return err + } + return nil +} + func loadReceipt(receiptsRow *sql.Row) (core.Receipt, error) { var contractAddress string var txHash string diff --git a/pkg/repositories/repository.go b/pkg/repositories/repository.go index df9a8172..7e4b7cc4 100644 --- a/pkg/repositories/repository.go +++ b/pkg/repositories/repository.go @@ -1,6 +1,9 @@ package repositories -import "github.com/8thlight/vulcanizedb/pkg/core" +import ( + "github.com/8thlight/vulcanizedb/pkg/core" + "github.com/8thlight/vulcanizedb/pkg/filters" +) const ( blocksFromHeadBeforeFinal = 20 @@ -19,4 +22,5 @@ type Repository interface { CreateLogs(log []core.Log) error FindLogs(address string, blockNumber int64) []core.Log SetBlocksStatus(chainHead int64) + AddFilter(filter filters.LogFilter) error } diff --git a/pkg/repositories/testing/helpers.go b/pkg/repositories/testing/helpers.go index c258f041..ed8ea710 100644 --- a/pkg/repositories/testing/helpers.go +++ b/pkg/repositories/testing/helpers.go @@ -7,6 +7,7 @@ import ( "math/big" "github.com/8thlight/vulcanizedb/pkg/core" + "github.com/8thlight/vulcanizedb/pkg/filters" "github.com/8thlight/vulcanizedb/pkg/repositories" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -18,6 +19,7 @@ func ClearData(postgres repositories.Postgres) { postgres.Db.MustExec("DELETE FROM blocks") postgres.Db.MustExec("DELETE FROM logs") postgres.Db.MustExec("DELETE FROM receipts") + postgres.Db.MustExec("DELETE FROM log_filters") } func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories.Repository) { @@ -600,6 +602,43 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Expect(err).To(HaveOccurred()) Expect(receipt).To(BeZero()) }) + }) + Describe("LogFilter", func() { + + It("inserts filter into watched events", func() { + + logFilter := filters.LogFilter{ + Name: "TestFilter", + FromBlock: 1, + ToBlock: 2, + Address: "0x8888f1f195afa192cfee860698584c030f4c9db1", + Topics: core.Topics{ + "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "", + "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "", + }, + } + err := repository.AddFilter(logFilter) + Expect(err).ToNot(HaveOccurred()) + }) + + It("returns error if name is not provided", func() { + + logFilter := filters.LogFilter{ + FromBlock: 1, + ToBlock: 2, + Address: "0x8888f1f195afa192cfee860698584c030f4c9db1", + Topics: core.Topics{ + "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "", + "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "", + }, + } + err := repository.AddFilter(logFilter) + Expect(err).To(HaveOccurred()) + }) }) } From 572023cdf53952f067874a5693882f039c4d13fe Mon Sep 17 00:00:00 2001 From: Matt K <1036969+mkrump@users.noreply.github.com> Date: Thu, 25 Jan 2018 13:21:55 -0600 Subject: [PATCH 15/22] Commandline (#135) * Add cmd line tool and Makefile * Add shared utils pkg * Add cmdline README * Update godo for new structure --- .gitignore | 1 + Gododir/main.go | 6 +- Gopkg.lock | 82 ++++++++++++++++- Gopkg.toml | 4 + Makefile | 36 ++++++++ cmd/add_filter/main.go | 8 +- cmd/populate_blocks/main.go | 6 +- cmd/run/main.go | 6 +- cmd/show_contract_summary/main.go | 8 +- cmd/vulcanize_db/main.go | 6 +- cmd/watch_contract/main.go | 8 +- commands/README.md | 90 +++++++++++++++++++ commands/cmd/addFilter.go | 75 ++++++++++++++++ commands/cmd/root.go | 74 +++++++++++++++ commands/cmd/sync.go | 77 ++++++++++++++++ commands/main.go | 9 ++ ...1516653373_add_watched_event_logs.down.sql | 3 +- {cmd => utils}/utils.go | 7 +- 18 files changed, 477 insertions(+), 29 deletions(-) create mode 100644 Makefile create mode 100644 commands/README.md create mode 100644 commands/cmd/addFilter.go create mode 100644 commands/cmd/root.go create mode 100644 commands/cmd/sync.go create mode 100644 commands/main.go rename {cmd => utils}/utils.go (95%) diff --git a/.gitignore b/.gitignore index f5b5e5ae..ffe62712 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ Vagrantfile vagrant_bootstrap.sh .vagrant test_scripts/ +vulcanize diff --git a/Gododir/main.go b/Gododir/main.go index 15200298..5f83012d 100644 --- a/Gododir/main.go +++ b/Gododir/main.go @@ -5,8 +5,8 @@ import ( "fmt" - "github.com/8thlight/vulcanizedb/cmd" "github.com/8thlight/vulcanizedb/pkg/config" + "github.com/8thlight/vulcanizedb/utils" do "gopkg.in/godo.v2" ) @@ -79,7 +79,7 @@ func tasks(p *do.Project) { p.Task("migrate", nil, func(context *do.Context) { environment := parseEnvironment(context) - cfg := cmd.LoadConfig(environment) + cfg := utils.LoadConfig(environment) connectString := config.DbConnectionString(cfg.Database) migrate := fmt.Sprintf("migrate -database '%s' -path ./db/migrations up", connectString) dumpSchema := fmt.Sprintf("pg_dump -O -s %s > db/schema.sql", cfg.Database.Name) @@ -89,7 +89,7 @@ func tasks(p *do.Project) { p.Task("rollback", nil, func(context *do.Context) { environment := parseEnvironment(context) - cfg := cmd.LoadConfig(environment) + cfg := utils.LoadConfig(environment) connectString := config.DbConnectionString(cfg.Database) migrate := fmt.Sprintf("migrate -database '%s' -path ./db/migrations down 1", connectString) dumpSchema := fmt.Sprintf("pg_dump -O -s %s > db/schema.sql", cfg.Database.Name) diff --git a/Gopkg.lock b/Gopkg.lock index 95a08ce7..a7bbf097 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -31,6 +31,12 @@ revision = "4bb3c89d44e372e6a9ab85a8be0c9345265c763a" version = "v1.7.3" +[[projects]] + name = "github.com/fsnotify/fsnotify" + packages = ["."] + revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" + version = "v1.4.7" + [[projects]] name = "github.com/go-stack/stack" packages = ["."] @@ -49,6 +55,12 @@ packages = ["."] revision = "553a641470496b2327abcac10b36396bd98e45c9" +[[projects]] + branch = "master" + name = "github.com/hashicorp/hcl" + packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"] + revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" + [[projects]] branch = "master" name = "github.com/howeyc/gopass" @@ -61,6 +73,12 @@ packages = [".","dcps/internetgateway1","dcps/internetgateway2","httpu","scpd","soap","ssdp"] revision = "dceda08e705b2acee36aab47d765ed801f64cfc7" +[[projects]] + name = "github.com/inconshreveable/mousetrap" + packages = ["."] + revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" + version = "v1.0" + [[projects]] name = "github.com/jackpal/go-nat-pmp" packages = ["."] @@ -79,6 +97,12 @@ packages = [".","oid"] revision = "83612a56d3dd153a94a629cd64925371c9adad78" +[[projects]] + name = "github.com/magiconair/properties" + packages = ["."] + revision = "d419a98cdbed11a922bf76f257b7c4be79b50e73" + version = "v1.7.4" + [[projects]] name = "github.com/mattn/go-colorable" packages = ["."] @@ -115,6 +139,18 @@ revision = "00c06406c2dd2e011f153a6502a21473676db33f" version = "v1.0.0" +[[projects]] + branch = "master" + name = "github.com/mitchellh/go-homedir" + packages = ["."] + revision = "b8bc1bf767474819792c23f32d8286a45736f1c6" + +[[projects]] + branch = "master" + name = "github.com/mitchellh/mapstructure" + packages = ["."] + revision = "b4575eea38cca1123ec2dc90c26529b5c5acfcff" + [[projects]] name = "github.com/nozzle/throttler" packages = ["."] @@ -133,6 +169,12 @@ revision = "c893efa28eb45626cdaa76c9f653b62488858837" version = "v1.2.0" +[[projects]] + name = "github.com/pelletier/go-toml" + packages = ["."] + revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8" + version = "v1.1.0" + [[projects]] branch = "master" name = "github.com/rcrowley/go-metrics" @@ -145,6 +187,42 @@ revision = "7af7a1e09ba336d2ea14b1ce73bf693c6837dbf6" version = "v1.2" +[[projects]] + name = "github.com/spf13/afero" + packages = [".","mem"] + revision = "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c" + version = "v1.0.2" + +[[projects]] + name = "github.com/spf13/cast" + packages = ["."] + revision = "acbeb36b902d72a7a4c18e8f3241075e7ab763e4" + version = "v1.1.0" + +[[projects]] + name = "github.com/spf13/cobra" + packages = ["."] + revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b" + version = "v0.0.1" + +[[projects]] + branch = "master" + name = "github.com/spf13/jwalterweatherman" + packages = ["."] + revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394" + +[[projects]] + name = "github.com/spf13/pflag" + packages = ["."] + revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66" + version = "v1.0.0" + +[[projects]] + name = "github.com/spf13/viper" + packages = ["."] + revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7" + version = "v1.0.0" + [[projects]] branch = "master" name = "github.com/syndtr/goleveldb" @@ -172,7 +250,7 @@ [[projects]] branch = "master" name = "golang.org/x/text" - packages = ["encoding","encoding/charmap","encoding/htmlindex","encoding/internal","encoding/internal/identifier","encoding/japanese","encoding/korean","encoding/simplifiedchinese","encoding/traditionalchinese","encoding/unicode","internal/gen","internal/tag","internal/utf8internal","language","runes","transform","unicode/cldr"] + packages = ["encoding","encoding/charmap","encoding/htmlindex","encoding/internal","encoding/internal/identifier","encoding/japanese","encoding/korean","encoding/simplifiedchinese","encoding/traditionalchinese","encoding/unicode","internal/gen","internal/tag","internal/triegen","internal/ucd","internal/utf8internal","language","runes","transform","unicode/cldr","unicode/norm"] revision = "be25de41fadfae372d6470bda81ca6beb55ef551" [[projects]] @@ -208,6 +286,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "2d7b9c5c88a94f3384b0cd754d35a3d7822a5858f439aaafe8c6477fb7c24f63" + inputs-digest = "46080c5bb453cb31ea731741d5defda225e2dc75bf7c11179d69422c82de37ca" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 0b5b260c..45a5aa07 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -32,3 +32,7 @@ [[constraint]] branch = "master" name = "github.com/lib/pq" + +[[constraint]] + name = "github.com/spf13/cobra" + version = "0.0.1" diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..165ee223 --- /dev/null +++ b/Makefile @@ -0,0 +1,36 @@ +HOST_NAME ?= $(DB_HOST_NAME) +PORT ?= $(DB_PORT) +NAME ?= $(DB_NAME) +CONNECT_STRING=postgresql://$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable + +$(MATTESMIGRATE): + go get -u -d github.com/mattes/migrate/cli github.com/lib/pq + go build -tags 'postgres' -o /usr/local/bin/migrate github.com/mattes/migrate/cli + +$(DEP): + go get -u github.com/golang/dep/cmd/dep + +$(GINKGO): + go get -u github.com/onsi/ginkgo/ginkgo + +checkdbvars: + test -n "$(HOST_NAME)" # $$HOST_NAME + test -n "$(PORT)" # $$PORT + test -n "$(NAME)" # $$NAME + +rollback: checkdbvars + migrate -database $(CONNECT_STRING) -path ./db/migrations down 1 + pg_dump -O -s $(CONNECT_STRING) > db/schema.sql + +migrate: $(MATTESMIGRATE) checkdbvars + migrate -database $(CONNECT_STRING) -path ./db/migrations up + pg_dump -O -s $(CONNECT_STRING) > db/schema.sql + +dep: $(DEP) + dep ensure + +build: dep + go build -o vulcanize ./commands + +test: $(GINKGO) + ginkgo -r diff --git a/cmd/add_filter/main.go b/cmd/add_filter/main.go index 026c0854..ac8f2655 100644 --- a/cmd/add_filter/main.go +++ b/cmd/add_filter/main.go @@ -6,9 +6,9 @@ import ( "io/ioutil" "log" - "github.com/8thlight/vulcanizedb/cmd" "github.com/8thlight/vulcanizedb/pkg/filters" "github.com/8thlight/vulcanizedb/pkg/geth" + "github.com/8thlight/vulcanizedb/utils" ) func main() { @@ -17,10 +17,10 @@ func main() { flag.Parse() var logFilters filters.LogFilters - config := cmd.LoadConfig(*environment) + config := utils.LoadConfig(*environment) blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) - absFilePath := cmd.AbsFilePath(*filterFilePath) + repository := utils.LoadPostgres(config.Database, blockchain.Node()) + absFilePath := utils.AbsFilePath(*filterFilePath) logFilterBytes, err := ioutil.ReadFile(absFilePath) if err != nil { log.Fatal(err) diff --git a/cmd/populate_blocks/main.go b/cmd/populate_blocks/main.go index 966b98e6..e367bf46 100644 --- a/cmd/populate_blocks/main.go +++ b/cmd/populate_blocks/main.go @@ -5,18 +5,18 @@ import ( "fmt" - "github.com/8thlight/vulcanizedb/cmd" "github.com/8thlight/vulcanizedb/pkg/geth" "github.com/8thlight/vulcanizedb/pkg/history" + "github.com/8thlight/vulcanizedb/utils" ) func main() { environment := flag.String("environment", "", "Environment name") startingBlockNumber := flag.Int("starting-number", -1, "First block to fill from") flag.Parse() - config := cmd.LoadConfig(*environment) + config := utils.LoadConfig(*environment) blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) + repository := utils.LoadPostgres(config.Database, blockchain.Node()) numberOfBlocksCreated := history.PopulateMissingBlocks(blockchain, repository, int64(*startingBlockNumber)) fmt.Printf("Populated %d blocks", numberOfBlocksCreated) } diff --git a/cmd/run/main.go b/cmd/run/main.go index a8fe9152..f95d3fe4 100644 --- a/cmd/run/main.go +++ b/cmd/run/main.go @@ -7,9 +7,9 @@ import ( "os" - "github.com/8thlight/vulcanizedb/cmd" "github.com/8thlight/vulcanizedb/pkg/geth" "github.com/8thlight/vulcanizedb/pkg/history" + "github.com/8thlight/vulcanizedb/utils" ) const ( @@ -23,9 +23,9 @@ func main() { ticker := time.NewTicker(pollingInterval) defer ticker.Stop() - config := cmd.LoadConfig(*environment) + config := utils.LoadConfig(*environment) blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) + repository := utils.LoadPostgres(config.Database, blockchain.Node()) validator := history.NewBlockValidator(blockchain, repository, 15) for range ticker.C { diff --git a/cmd/show_contract_summary/main.go b/cmd/show_contract_summary/main.go index 23ff73d1..1f03d15e 100644 --- a/cmd/show_contract_summary/main.go +++ b/cmd/show_contract_summary/main.go @@ -9,9 +9,9 @@ import ( "strings" - "github.com/8thlight/vulcanizedb/cmd" "github.com/8thlight/vulcanizedb/pkg/contract_summary" "github.com/8thlight/vulcanizedb/pkg/geth" + "github.com/8thlight/vulcanizedb/utils" ) func main() { @@ -21,10 +21,10 @@ func main() { flag.Parse() contractHashLowered := strings.ToLower(*contractHash) - config := cmd.LoadConfig(*environment) + config := utils.LoadConfig(*environment) blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) - blockNumber := cmd.RequestedBlockNumber(_blockNumber) + repository := utils.LoadPostgres(config.Database, blockchain.Node()) + blockNumber := utils.RequestedBlockNumber(_blockNumber) contractSummary, err := contract_summary.NewSummary(blockchain, repository, contractHashLowered, blockNumber) if err != nil { diff --git a/cmd/vulcanize_db/main.go b/cmd/vulcanize_db/main.go index 24cf0776..366f8615 100644 --- a/cmd/vulcanize_db/main.go +++ b/cmd/vulcanize_db/main.go @@ -7,11 +7,11 @@ import ( "os" - "github.com/8thlight/vulcanizedb/cmd" "github.com/8thlight/vulcanizedb/pkg/core" "github.com/8thlight/vulcanizedb/pkg/geth" "github.com/8thlight/vulcanizedb/pkg/history" "github.com/8thlight/vulcanizedb/pkg/repositories" + "github.com/8thlight/vulcanizedb/utils" ) const ( @@ -32,9 +32,9 @@ func main() { ticker := time.NewTicker(pollingInterval) defer ticker.Stop() - config := cmd.LoadConfig(*environment) + config := utils.LoadConfig(*environment) blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) + repository := utils.LoadPostgres(config.Database, blockchain.Node()) validator := history.NewBlockValidator(blockchain, repository, 15) missingBlocksPopulated := make(chan int) diff --git a/cmd/watch_contract/main.go b/cmd/watch_contract/main.go index 0a29f552..1a02b52d 100644 --- a/cmd/watch_contract/main.go +++ b/cmd/watch_contract/main.go @@ -5,9 +5,9 @@ import ( "strings" - "github.com/8thlight/vulcanizedb/cmd" "github.com/8thlight/vulcanizedb/pkg/core" "github.com/8thlight/vulcanizedb/pkg/geth" + "github.com/8thlight/vulcanizedb/utils" ) func main() { @@ -19,10 +19,10 @@ func main() { flag.Parse() contractHashLowered := strings.ToLower(*contractHash) - contractAbiString := cmd.GetAbi(*abiFilepath, contractHashLowered, *network) - config := cmd.LoadConfig(*environment) + contractAbiString := utils.GetAbi(*abiFilepath, contractHashLowered, *network) + config := utils.LoadConfig(*environment) blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) + repository := utils.LoadPostgres(config.Database, blockchain.Node()) watchedContract := core.Contract{ Abi: contractAbiString, Hash: contractHashLowered, diff --git a/commands/README.md b/commands/README.md new file mode 100644 index 00000000..5349be62 --- /dev/null +++ b/commands/README.md @@ -0,0 +1,90 @@ +# Vulcanize DB + +[![Build Status](https://travis-ci.com/8thlight/vulcanizedb.svg?token=GKv2Y33qsFnfYgejjvYx&branch=master)](https://travis-ci.com/8thlight/vulcanizedb) + +## Development Setup + +### Dependencies + + - Go 1.9+ + - Postgres 10 + - Go Ethereum + - https://ethereum.github.io/go-ethereum/downloads/ + +### Cloning the Repository + +1. `git config --global url."git@github.com:".insteadOf "https://github.com/"` + - By default, `go get` does not work for private GitHub repos. This will fix that. +2. `go get github.com/8thlight/vulcanizedb` +3. `cd $GOPATH/src/github.com/8thlight/vulcanizedb` +4. `dep ensure` + +### Setting up the Databases + +1. Install Postgres +2. Create a superuser for yourself and make sure `psql --list` works without prompting for a password. +3. `createdb vulcanize_private` +4. `cd $GOPATH/src/github.com/8thlight/vulcanizedb` +5. `HOST_NAME=localhost NAME=vulcanize_public PORT=5432 make migrate` + * See below for configuring additional environments + +Adding a new migration: `./scripts/create_migration ` + +### Building +1. `make build` + +### Creating/Using a Private Blockchain + +Syncing the public blockchain takes many hours for the initial sync and will download 20+ GB of data. +Here are some instructions for creating a private blockchain that does not depend on having a network connection. + +1. Run `./scripts/setup` to create a private blockchain with a new account. + * This will result in a warning. +2. Run `./scripts/start_private_blockchain`. +3. Run `godo run -- --environment=private` to start listener. + +### Connecting to the Public Blockchain + +`./scripts/start_blockchain` + +### IPC File Paths + +The default location for Ethereum is: + - `$HOME/Library/Ethereum` for Mac + - `$HOME/.ethereum` for Ubuntu + - `$GOPATH/src/gihub.com/8thlight/vulcanizedb/test_data_dir/geth.ipc` for private blockchain. + +**Note the location of the ipc file is outputted when you connect to geth. It is needed to for configuration** + +## Start Vulcanize DB +1. Start geth +2. In a separate terminal start vulcanize_db + - `vulcanize sync --config --starting-block-number ` + +## Watch contract events +1. Start geth +2. In a separate terminal start vulcanize_db + - `vulcanize sync --config --starting-block-number ` +3. Create event filter + - `vulcanize addFilter --config --filter-filepath ` + +### Configuring Additional Environments + +You can create configuration files for additional environments. + + * Among other things, it will require the IPC file path + * See `environments/private.toml` for an example + * You will need to do this if you want to run a node connecting to the public blockchain + +## Running the Tests + +### Unit Tests + +1. `go test ./pkg/...` + +### Integration Test + +In order to run the integration tests, you will need to run them against a real blockchain. At the moment the integration tests require [Geth v1.7.2](https://ethereum.github.io/go-ethereum/downloads/) as they depend on the `--dev` mode, which changed in v1.7.3 + +1. Run `./scripts/start_private_blockchain` as a separate process. +2. `go test ./...` to run all tests. diff --git a/commands/cmd/addFilter.go b/commands/cmd/addFilter.go new file mode 100644 index 00000000..21318429 --- /dev/null +++ b/commands/cmd/addFilter.go @@ -0,0 +1,75 @@ +package cmd + +import ( + "encoding/json" + "io/ioutil" + "log" + + "github.com/8thlight/vulcanizedb/pkg/filters" + "github.com/8thlight/vulcanizedb/pkg/geth" + "github.com/8thlight/vulcanizedb/utils" + "github.com/spf13/cobra" +) + +// addFilterCmd represents the addFilter command +var addFilterCmd = &cobra.Command{ + Use: "addFilter", + Short: "Adds event filter to vulcanize_db", + Long: `An event filter is added to the vulcanize_db. +All events matching the filter conitions will be tracked +in vulcanize. + +vulcanize addFilter --config config.toml --filter-filepath filter.json + +The event filters are expected to match +the format described in the ethereum RPC wiki: + +https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_newfilter + +[{ + "fromBlock": "0x1", + "toBlock": "0x2", + "address": "0x8888f1f195afa192cfee860698584c030f4c9db1", + "topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + null, + "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "0x0000000000000000000000000aff3454fce5edbc8cca8697c15331677e6ebccc"] +}] +`, + Run: func(cmd *cobra.Command, args []string) { + addFilter() + }, +} + +var filterFilepath string + +func init() { + rootCmd.AddCommand(addFilterCmd) + + addFilterCmd.PersistentFlags().StringVar(&filterFilepath, "filter-filepath", "", "path/to/filter.json") + addFilterCmd.MarkFlagRequired("filter-filepath") +} + +func addFilter() { + if filterFilepath == "" { + log.Fatal("filter-filepath required") + } + var logFilters filters.LogFilters + blockchain := geth.NewBlockchain(ipc) + repository := utils.LoadPostgres(databaseConfig, blockchain.Node()) + absFilePath := utils.AbsFilePath(filterFilepath) + logFilterBytes, err := ioutil.ReadFile(absFilePath) + if err != nil { + log.Fatal(err) + } + err = json.Unmarshal(logFilterBytes, &logFilters) + if err != nil { + log.Fatal(err) + } + for _, filter := range logFilters { + err = repository.AddFilter(filter) + if err != nil { + log.Fatal(err) + } + } +} diff --git a/commands/cmd/root.go b/commands/cmd/root.go new file mode 100644 index 00000000..69be2a30 --- /dev/null +++ b/commands/cmd/root.go @@ -0,0 +1,74 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/8thlight/vulcanizedb/pkg/config" + "github.com/mitchellh/go-homedir" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var cfgFile string +var databaseConfig config.Database +var ipc string + +var rootCmd = &cobra.Command{ + Use: "vulcanize", + PersistentPreRun: database, +} + +func Execute() { + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func database(cmd *cobra.Command, args []string) { + ipc = viper.GetString("client.ipcpath") + databaseConfig = config.Database{ + Name: viper.GetString("database.name"), + Hostname: viper.GetString("database.hostname"), + Port: viper.GetInt("database.port"), + } + viper.Set("database.config", databaseConfig) +} + +func init() { + cobra.OnInitialize(initConfig) + + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file: (default is $HOME/.vulcanize.yaml)") + rootCmd.PersistentFlags().String("database-name", "vulcanize", "database: name") + rootCmd.PersistentFlags().Int("database-port", 5432, "database: port") + rootCmd.PersistentFlags().String("database-hostname", "localhost", "database: hostname") + rootCmd.PersistentFlags().String("client-ipcPath", "", "geth: geth.ipc file") + + viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name")) + viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port")) + viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname")) + viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath")) + +} + +func initConfig() { + if cfgFile != "" { + viper.SetConfigFile(cfgFile) + } else { + home, err := homedir.Dir() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + viper.AddConfigPath(home) + viper.SetConfigName(".vulcanize") + } + + viper.AutomaticEnv() + + if err := viper.ReadInConfig(); err == nil { + fmt.Printf("Using config file: %s\n\n", viper.ConfigFileUsed()) + } +} diff --git a/commands/cmd/sync.go b/commands/cmd/sync.go new file mode 100644 index 00000000..214d0ce0 --- /dev/null +++ b/commands/cmd/sync.go @@ -0,0 +1,77 @@ +package cmd + +import ( + "os" + + "time" + + "github.com/8thlight/vulcanizedb/pkg/core" + "github.com/8thlight/vulcanizedb/pkg/geth" + "github.com/8thlight/vulcanizedb/pkg/history" + "github.com/8thlight/vulcanizedb/pkg/repositories" + "github.com/8thlight/vulcanizedb/utils" + "github.com/spf13/cobra" +) + +// syncCmd represents the sync command +var syncCmd = &cobra.Command{ + Use: "sync", + Short: "Syncs vulcanize_db with local ethereum node", + Long: `Syncs vulcanize_db with local ethereum node. +vulcanize sync --startingBlockNumber 0 --config public.toml + +Expects ethereum node to be running and requires a .toml config: + + [database] + name = "vulcanize_public" + hostname = "localhost" + port = 5432 + + [client] + ipcPath = "/Users/mattkrump/Library/Ethereum/geth.ipc" +`, + Run: func(cmd *cobra.Command, args []string) { + sync() + }, +} + +const ( + pollingInterval = 7 * time.Second +) + +var startingBlockNumber int + +func init() { + rootCmd.AddCommand(syncCmd) + + syncCmd.Flags().IntVarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from") +} + +func backFillAllBlocks(blockchain core.Blockchain, repository repositories.Postgres, missingBlocksPopulated chan int, startingBlockNumber int64) { + go func() { + missingBlocksPopulated <- history.PopulateMissingBlocks(blockchain, repository, startingBlockNumber) + }() +} + +func sync() { + ticker := time.NewTicker(pollingInterval) + defer ticker.Stop() + + blockchain := geth.NewBlockchain(ipc) + repository := utils.LoadPostgres(databaseConfig, blockchain.Node()) + validator := history.NewBlockValidator(blockchain, repository, 15) + + missingBlocksPopulated := make(chan int) + _startingBlockNumber := int64(startingBlockNumber) + go backFillAllBlocks(blockchain, repository, missingBlocksPopulated, _startingBlockNumber) + + for { + select { + case <-ticker.C: + window := validator.ValidateBlocks() + validator.Log(os.Stdout, window) + case <-missingBlocksPopulated: + go backFillAllBlocks(blockchain, repository, missingBlocksPopulated, _startingBlockNumber) + } + } +} diff --git a/commands/main.go b/commands/main.go new file mode 100644 index 00000000..50e90e3b --- /dev/null +++ b/commands/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/8thlight/vulcanizedb/commands/cmd" +) + +func main() { + cmd.Execute() +} diff --git a/db/migrations/1516653373_add_watched_event_logs.down.sql b/db/migrations/1516653373_add_watched_event_logs.down.sql index 14bb2987..294d7bfe 100644 --- a/db/migrations/1516653373_add_watched_event_logs.down.sql +++ b/db/migrations/1516653373_add_watched_event_logs.down.sql @@ -1 +1,2 @@ -DROP VIEW watched_event_logs; \ No newline at end of file +DROP VIEW watched_event_logs; +DROP VIEW block_stats; diff --git a/cmd/utils.go b/utils/utils.go similarity index 95% rename from cmd/utils.go rename to utils/utils.go index 03a6be65..ad091437 100644 --- a/cmd/utils.go +++ b/utils/utils.go @@ -1,4 +1,4 @@ -package cmd +package utils import ( "log" @@ -7,6 +7,8 @@ import ( "math/big" + "os" + "github.com/8thlight/vulcanizedb/pkg/config" "github.com/8thlight/vulcanizedb/pkg/core" "github.com/8thlight/vulcanizedb/pkg/geth" @@ -40,7 +42,8 @@ func ReadAbiFile(abiFilepath string) string { func AbsFilePath(filePath string) string { if !filepath.IsAbs(filePath) { - filePath = filepath.Join(config.ProjectRoot(), filePath) + cwd, _ := os.Getwd() + filePath = filepath.Join(cwd, filePath) } return filePath } From d09c2ae9bb89954d2dda43cb20d4f390e12af0fe Mon Sep 17 00:00:00 2001 From: Matt K <1036969+mkrump@users.noreply.github.com> Date: Thu, 25 Jan 2018 15:46:55 -0600 Subject: [PATCH 16/22] Remove godo (#136) --- .gitignore | 2 +- .travis.yml | 5 +- Gododir/main.go | 118 ----------------------------- Makefile | 16 ++-- README.md | 116 ++++++++++------------------ {commands/cmd => cmd}/addFilter.go | 6 +- cmd/add_filter/main.go | 38 ---------- cmd/populate_blocks/main.go | 22 ------ {commands/cmd => cmd}/root.go | 14 ++-- cmd/run/main.go | 35 --------- cmd/show_contract_summary/main.go | 35 --------- {commands/cmd => cmd}/sync.go | 6 +- cmd/vulcanize_db/main.go | 53 ------------- cmd/watch_contract/main.go | 31 -------- commands/README.md | 90 ---------------------- db/schema.sql | 8 ++ commands/main.go => main.go | 2 +- 17 files changed, 74 insertions(+), 523 deletions(-) delete mode 100644 Gododir/main.go rename {commands/cmd => cmd}/addFilter.go (93%) delete mode 100644 cmd/add_filter/main.go delete mode 100644 cmd/populate_blocks/main.go rename {commands/cmd => cmd}/root.go (77%) delete mode 100644 cmd/run/main.go delete mode 100644 cmd/show_contract_summary/main.go rename {commands/cmd => cmd}/sync.go (91%) delete mode 100644 cmd/vulcanize_db/main.go delete mode 100644 cmd/watch_contract/main.go delete mode 100644 commands/README.md rename commands/main.go => main.go (54%) diff --git a/.gitignore b/.gitignore index ffe62712..55774619 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,4 @@ Vagrantfile vagrant_bootstrap.sh .vagrant test_scripts/ -vulcanize +vulcanizedb diff --git a/.travis.yml b/.travis.yml index 5b51065f..c3c9b8d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,11 +8,8 @@ addons: postgresql: "9.6" before_install: - # godo - - go get -u gopkg.in/godo.v2/cmd/godo # dep - go get -u github.com/golang/dep/cmd/dep - - dep ensure # ginkgo - go get -u github.com/onsi/ginkgo/ginkgo # migrate @@ -30,7 +27,7 @@ install: before_script: - ./scripts/setup - nohup ./scripts/start_private_blockchain db/schema.sql", cfg.Database.Name) - context.Bash(migrate) - context.Bash(dumpSchema) - }) - - p.Task("rollback", nil, func(context *do.Context) { - environment := parseEnvironment(context) - cfg := utils.LoadConfig(environment) - connectString := config.DbConnectionString(cfg.Database) - migrate := fmt.Sprintf("migrate -database '%s' -path ./db/migrations down 1", connectString) - dumpSchema := fmt.Sprintf("pg_dump -O -s %s > db/schema.sql", cfg.Database.Name) - context.Bash(migrate) - context.Bash(dumpSchema) - }) - - p.Task("showContractSummary", nil, func(context *do.Context) { - environment := parseEnvironment(context) - contractHash := context.Args.MayString("", "contract-hash", "c") - blockNumber := context.Args.MayInt(-1, "block-number", "b") - if contractHash == "" { - log.Fatalln("--contract-hash required") - } - context.Start(`go run main.go --environment={{.environment}} --contract-hash={{.contractHash}} --block-number={{.blockNumber}}`, - do.M{"environment": environment, - "contractHash": contractHash, - "blockNumber": blockNumber, - "$in": "cmd/show_contract_summary"}) - }) - -} - -func main() { - do.Godo(tasks) -} diff --git a/Makefile b/Makefile index 165ee223..43a43a5a 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ -HOST_NAME ?= $(DB_HOST_NAME) -PORT ?= $(DB_PORT) -NAME ?= $(DB_NAME) +HOST_NAME = +PORT = +NAME = CONNECT_STRING=postgresql://$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable $(MATTESMIGRATE): @@ -14,7 +14,7 @@ $(GINKGO): go get -u github.com/onsi/ginkgo/ginkgo checkdbvars: - test -n "$(HOST_NAME)" # $$HOST_NAME + test -n "$(HOST_NAME)" # $$HOST_NAME test -n "$(PORT)" # $$PORT test -n "$(NAME)" # $$NAME @@ -26,11 +26,15 @@ migrate: $(MATTESMIGRATE) checkdbvars migrate -database $(CONNECT_STRING) -path ./db/migrations up pg_dump -O -s $(CONNECT_STRING) > db/schema.sql +import: + test -n "$(NAME)" # $$NAME + psql $(NAME) < db/schema.sql + dep: $(DEP) dep ensure -build: dep - go build -o vulcanize ./commands +build: dep + go build test: $(GINKGO) ginkgo -r diff --git a/README.md b/README.md index b8dc09e3..adb9a3c8 100644 --- a/README.md +++ b/README.md @@ -2,20 +2,49 @@ [![Build Status](https://travis-ci.com/8thlight/vulcanizedb.svg?token=GKv2Y33qsFnfYgejjvYx&branch=master)](https://travis-ci.com/8thlight/vulcanizedb) -## Development Setup - ### Dependencies - Go 1.9+ - - https://github.com/golang/dep - - `go get -u github.com/golang/dep/cmd/dep` - - https://github.com/go-godo/godo - - `go get -u gopkg.in/godo.v2/cmd/godo` - Postgres 10 - Go Ethereum - https://ethereum.github.io/go-ethereum/downloads/ + +### Installation + go get github.com/8thlight/vulcanizedb -### Cloning the Repository +### Setting up the Databases + +1. Install Postgres +2. Create a superuser for yourself and make sure `psql --list` works without prompting for a password. +3. `createdb vulcanize_private` +4. `cd $GOPATH/src/github.com/8thlight/vulcanizedb` +5. Import the schema: + `psql vulcanize_private < db/schema.sql` + or run the migrations: + `make migrate HOST_NAME=localhost NAME=vulcanize_public PORT=5432` + * See below for configuring additional environments + +Adding a new migration: `./scripts/create_migration ` + +## Start syncing with postgres +1. Start geth +2. In a separate terminal start vulcanize_db + - `vulcanizedb sync --config --starting-block-number ` + + * see `environments` for example config + +## Watch specific contract events +1. Start geth +2. In a separate terminal start vulcanize_db + - `vulcanizedb sync --config --starting-block-number ` +3. Create event filter + - `vulcanizedb addFilter --config --filter-filepath ` + + * see `filters` for example filter + +## Development Setup + +### Cloning the Repository (Private repo only) 1. `git config --global url."git@github.com:".insteadOf "https://github.com/"` - By default, `go get` does not work for private GitHub repos. This will fix that. @@ -23,32 +52,14 @@ 3. `cd $GOPATH/src/github.com/8thlight/vulcanizedb` 4. `dep ensure` -### Setting up the Databases +### Creating/Using a test node -1. Install Postgres -2. Create a superuser for yourself and make sure `psql --list` works without prompting for a password. -3. `go get -u -d github.com/mattes/migrate/cli github.com/lib/pq` -4. `go build -tags 'postgres' -o /usr/local/bin/migrate github.com/mattes/migrate/cli` -5. `createdb vulcanize_private` -6. `cd $GOPATH/src/github.com/8thlight/vulcanizedb` -7. `godo migrate -- --environment=` - * See below for configuring additional environments - -Adding a new migration: `./scripts/create_migration ` - -### Creating/Using a Private Blockchain - -Syncing the public blockchain takes many hours for the initial sync and will download 20+ GB of data. -Here are some instructions for creating a private blockchain that does not depend on having a network connection. +Syncing the against the public network takes many hours for the initial sync and will download 20+ GB of data. +Here are some instructions for creating a private test node that does not depend on having a network connection. 1. Run `./scripts/setup` to create a private blockchain with a new account. * This will result in a warning. 2. Run `./scripts/start_private_blockchain`. -3. Run `godo run -- --environment=private` to start listener. - -### Connecting to the Public Blockchain - -`./scripts/start_blockchain` ### IPC File Paths @@ -57,54 +68,7 @@ The default location for Ethereum is: - `$HOME/.ethereum` for Ubuntu - `$GOPATH/src/gihub.com/8thlight/vulcanizedb/test_data_dir/geth.ipc` for private blockchain. -**Note the location of the ipc file is outputted when you connect to a blockchain. It is needed to for configuration** - -## Start Vulcanize DB -1. Start geth -2. In a separate terminal start vulcanize_db - - `godo vulcanizeDb -- --environment=` - -## Watch contract events -1. Start geth -2. In a separate terminal start vulcanize_db - - `godo vulcanizeDb -- --environment=` -3. Create event filter - - `godo watchEvent -- --environment= --filter-filepath=` - -## Running Listener - -1. Start geth -2. In a separate terminal start listener (ipcDir location) - - `godo run -- --environment=` - -## Retrieving Historical Data - -1. Start geth -2. In a separate terminal start listener (ipcDir location) - - `godo populateBlocks -- --environment= --starting-number=` - -## Retrieve Contract Attributes - -1. Add contract ABI to contracts / environment directory: -``` -vulcanizedb/ - contracts/ - public/ - .json - private/ -``` -The name of the JSON file should correspond the contract's address. - -2. Start watching the contract `godo watchContract -- --environment= --contract-hash=` -3. Request summary data `godo showContractSummary -- --environment= --contract-hash=` - -### Configuring Additional Environments - -You can create configuration files for additional environments. - - * Among other things, it will require the IPC file path - * See `environments/private.toml` for an example - * You will need to do this if you want to run a node connecting to the public blockchain +**Note the location of the ipc file is printed to the console when you start geth. It is needed to for configuration** ## Running the Tests diff --git a/commands/cmd/addFilter.go b/cmd/addFilter.go similarity index 93% rename from commands/cmd/addFilter.go rename to cmd/addFilter.go index 21318429..bfea9258 100644 --- a/commands/cmd/addFilter.go +++ b/cmd/addFilter.go @@ -14,12 +14,12 @@ import ( // addFilterCmd represents the addFilter command var addFilterCmd = &cobra.Command{ Use: "addFilter", - Short: "Adds event filter to vulcanize_db", + Short: "Adds event filter to vulcanizedb", Long: `An event filter is added to the vulcanize_db. All events matching the filter conitions will be tracked -in vulcanize. +in vulcanizedb. -vulcanize addFilter --config config.toml --filter-filepath filter.json +vulcanizedb addFilter --config config.toml --filter-filepath filter.json The event filters are expected to match the format described in the ethereum RPC wiki: diff --git a/cmd/add_filter/main.go b/cmd/add_filter/main.go deleted file mode 100644 index ac8f2655..00000000 --- a/cmd/add_filter/main.go +++ /dev/null @@ -1,38 +0,0 @@ -package main - -import ( - "encoding/json" - "flag" - "io/ioutil" - "log" - - "github.com/8thlight/vulcanizedb/pkg/filters" - "github.com/8thlight/vulcanizedb/pkg/geth" - "github.com/8thlight/vulcanizedb/utils" -) - -func main() { - environment := flag.String("environment", "", "Environment name") - filterFilePath := flag.String("filter-filepath", "", "path/to/filter.json") - - flag.Parse() - var logFilters filters.LogFilters - config := utils.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := utils.LoadPostgres(config.Database, blockchain.Node()) - absFilePath := utils.AbsFilePath(*filterFilePath) - logFilterBytes, err := ioutil.ReadFile(absFilePath) - if err != nil { - log.Fatal(err) - } - err = json.Unmarshal(logFilterBytes, &logFilters) - if err != nil { - log.Fatal(err) - } - for _, filter := range logFilters { - err = repository.AddFilter(filter) - if err != nil { - log.Fatal(err) - } - } -} diff --git a/cmd/populate_blocks/main.go b/cmd/populate_blocks/main.go deleted file mode 100644 index e367bf46..00000000 --- a/cmd/populate_blocks/main.go +++ /dev/null @@ -1,22 +0,0 @@ -package main - -import ( - "flag" - - "fmt" - - "github.com/8thlight/vulcanizedb/pkg/geth" - "github.com/8thlight/vulcanizedb/pkg/history" - "github.com/8thlight/vulcanizedb/utils" -) - -func main() { - environment := flag.String("environment", "", "Environment name") - startingBlockNumber := flag.Int("starting-number", -1, "First block to fill from") - flag.Parse() - config := utils.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := utils.LoadPostgres(config.Database, blockchain.Node()) - numberOfBlocksCreated := history.PopulateMissingBlocks(blockchain, repository, int64(*startingBlockNumber)) - fmt.Printf("Populated %d blocks", numberOfBlocksCreated) -} diff --git a/commands/cmd/root.go b/cmd/root.go similarity index 77% rename from commands/cmd/root.go rename to cmd/root.go index 69be2a30..ad5a1b9b 100644 --- a/commands/cmd/root.go +++ b/cmd/root.go @@ -15,7 +15,7 @@ var databaseConfig config.Database var ipc string var rootCmd = &cobra.Command{ - Use: "vulcanize", + Use: "vulcanizedb", PersistentPreRun: database, } @@ -39,11 +39,11 @@ func database(cmd *cobra.Command, args []string) { func init() { cobra.OnInitialize(initConfig) - rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file: (default is $HOME/.vulcanize.yaml)") - rootCmd.PersistentFlags().String("database-name", "vulcanize", "database: name") - rootCmd.PersistentFlags().Int("database-port", 5432, "database: port") - rootCmd.PersistentFlags().String("database-hostname", "localhost", "database: hostname") - rootCmd.PersistentFlags().String("client-ipcPath", "", "geth: geth.ipc file") + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "environment/public.toml", "config file location") + rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name") + rootCmd.PersistentFlags().Int("database-port", 5432, "database port") + rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname") + rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file") viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name")) viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port")) @@ -63,7 +63,7 @@ func initConfig() { } viper.AddConfigPath(home) - viper.SetConfigName(".vulcanize") + viper.SetConfigName(".vulcanizedb") } viper.AutomaticEnv() diff --git a/cmd/run/main.go b/cmd/run/main.go deleted file mode 100644 index f95d3fe4..00000000 --- a/cmd/run/main.go +++ /dev/null @@ -1,35 +0,0 @@ -package main - -import ( - "flag" - - "time" - - "os" - - "github.com/8thlight/vulcanizedb/pkg/geth" - "github.com/8thlight/vulcanizedb/pkg/history" - "github.com/8thlight/vulcanizedb/utils" -) - -const ( - pollingInterval = 7 * time.Second -) - -func main() { - environment := flag.String("environment", "", "Environment name") - flag.Parse() - - ticker := time.NewTicker(pollingInterval) - defer ticker.Stop() - - config := utils.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := utils.LoadPostgres(config.Database, blockchain.Node()) - validator := history.NewBlockValidator(blockchain, repository, 15) - - for range ticker.C { - window := validator.ValidateBlocks() - validator.Log(os.Stdout, window) - } -} diff --git a/cmd/show_contract_summary/main.go b/cmd/show_contract_summary/main.go deleted file mode 100644 index 1f03d15e..00000000 --- a/cmd/show_contract_summary/main.go +++ /dev/null @@ -1,35 +0,0 @@ -package main - -import ( - "flag" - - "log" - - "fmt" - - "strings" - - "github.com/8thlight/vulcanizedb/pkg/contract_summary" - "github.com/8thlight/vulcanizedb/pkg/geth" - "github.com/8thlight/vulcanizedb/utils" -) - -func main() { - environment := flag.String("environment", "", "Environment name") - contractHash := flag.String("contract-hash", "", "Contract hash to show summary") - _blockNumber := flag.Int64("block-number", -1, "Block number of summary") - flag.Parse() - - contractHashLowered := strings.ToLower(*contractHash) - config := utils.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := utils.LoadPostgres(config.Database, blockchain.Node()) - blockNumber := utils.RequestedBlockNumber(_blockNumber) - - contractSummary, err := contract_summary.NewSummary(blockchain, repository, contractHashLowered, blockNumber) - if err != nil { - log.Fatalln(err) - } - output := contract_summary.GenerateConsoleOutput(contractSummary) - fmt.Println(output) -} diff --git a/commands/cmd/sync.go b/cmd/sync.go similarity index 91% rename from commands/cmd/sync.go rename to cmd/sync.go index 214d0ce0..57408a42 100644 --- a/commands/cmd/sync.go +++ b/cmd/sync.go @@ -16,9 +16,9 @@ import ( // syncCmd represents the sync command var syncCmd = &cobra.Command{ Use: "sync", - Short: "Syncs vulcanize_db with local ethereum node", - Long: `Syncs vulcanize_db with local ethereum node. -vulcanize sync --startingBlockNumber 0 --config public.toml + Short: "Syncs vulcanizedb with local ethereum node", + Long: `Syncs vulcanizedb with local ethereum node. +vulcanizedb sync --startingBlockNumber 0 --config public.toml Expects ethereum node to be running and requires a .toml config: diff --git a/cmd/vulcanize_db/main.go b/cmd/vulcanize_db/main.go deleted file mode 100644 index 366f8615..00000000 --- a/cmd/vulcanize_db/main.go +++ /dev/null @@ -1,53 +0,0 @@ -package main - -import ( - "flag" - - "time" - - "os" - - "github.com/8thlight/vulcanizedb/pkg/core" - "github.com/8thlight/vulcanizedb/pkg/geth" - "github.com/8thlight/vulcanizedb/pkg/history" - "github.com/8thlight/vulcanizedb/pkg/repositories" - "github.com/8thlight/vulcanizedb/utils" -) - -const ( - pollingInterval = 7 * time.Second -) - -func backFillAllBlocks(blockchain core.Blockchain, repository repositories.Postgres, missingBlocksPopulated chan int, startingBlockNumber int64) { - go func() { - missingBlocksPopulated <- history.PopulateMissingBlocks(blockchain, repository, startingBlockNumber) - }() -} - -func main() { - environment := flag.String("environment", "", "Environment name") - startingBlockNumber := flag.Int("starting-number", 0, "First block to fill from") - flag.Parse() - - ticker := time.NewTicker(pollingInterval) - defer ticker.Stop() - - config := utils.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := utils.LoadPostgres(config.Database, blockchain.Node()) - validator := history.NewBlockValidator(blockchain, repository, 15) - - missingBlocksPopulated := make(chan int) - _startingBlockNumber := int64(*startingBlockNumber) - go backFillAllBlocks(blockchain, repository, missingBlocksPopulated, _startingBlockNumber) - - for { - select { - case <-ticker.C: - window := validator.ValidateBlocks() - validator.Log(os.Stdout, window) - case <-missingBlocksPopulated: - go backFillAllBlocks(blockchain, repository, missingBlocksPopulated, _startingBlockNumber) - } - } -} diff --git a/cmd/watch_contract/main.go b/cmd/watch_contract/main.go deleted file mode 100644 index 1a02b52d..00000000 --- a/cmd/watch_contract/main.go +++ /dev/null @@ -1,31 +0,0 @@ -package main - -import ( - "flag" - - "strings" - - "github.com/8thlight/vulcanizedb/pkg/core" - "github.com/8thlight/vulcanizedb/pkg/geth" - "github.com/8thlight/vulcanizedb/utils" -) - -func main() { - environment := flag.String("environment", "", "Environment name") - contractHash := flag.String("contract-hash", "", "contract-hash=x1234") - abiFilepath := flag.String("abi-filepath", "", "path/to/abifile.json") - network := flag.String("network", "", "ropsten") - - flag.Parse() - contractHashLowered := strings.ToLower(*contractHash) - - contractAbiString := utils.GetAbi(*abiFilepath, contractHashLowered, *network) - config := utils.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := utils.LoadPostgres(config.Database, blockchain.Node()) - watchedContract := core.Contract{ - Abi: contractAbiString, - Hash: contractHashLowered, - } - repository.CreateContract(watchedContract) -} diff --git a/commands/README.md b/commands/README.md deleted file mode 100644 index 5349be62..00000000 --- a/commands/README.md +++ /dev/null @@ -1,90 +0,0 @@ -# Vulcanize DB - -[![Build Status](https://travis-ci.com/8thlight/vulcanizedb.svg?token=GKv2Y33qsFnfYgejjvYx&branch=master)](https://travis-ci.com/8thlight/vulcanizedb) - -## Development Setup - -### Dependencies - - - Go 1.9+ - - Postgres 10 - - Go Ethereum - - https://ethereum.github.io/go-ethereum/downloads/ - -### Cloning the Repository - -1. `git config --global url."git@github.com:".insteadOf "https://github.com/"` - - By default, `go get` does not work for private GitHub repos. This will fix that. -2. `go get github.com/8thlight/vulcanizedb` -3. `cd $GOPATH/src/github.com/8thlight/vulcanizedb` -4. `dep ensure` - -### Setting up the Databases - -1. Install Postgres -2. Create a superuser for yourself and make sure `psql --list` works without prompting for a password. -3. `createdb vulcanize_private` -4. `cd $GOPATH/src/github.com/8thlight/vulcanizedb` -5. `HOST_NAME=localhost NAME=vulcanize_public PORT=5432 make migrate` - * See below for configuring additional environments - -Adding a new migration: `./scripts/create_migration ` - -### Building -1. `make build` - -### Creating/Using a Private Blockchain - -Syncing the public blockchain takes many hours for the initial sync and will download 20+ GB of data. -Here are some instructions for creating a private blockchain that does not depend on having a network connection. - -1. Run `./scripts/setup` to create a private blockchain with a new account. - * This will result in a warning. -2. Run `./scripts/start_private_blockchain`. -3. Run `godo run -- --environment=private` to start listener. - -### Connecting to the Public Blockchain - -`./scripts/start_blockchain` - -### IPC File Paths - -The default location for Ethereum is: - - `$HOME/Library/Ethereum` for Mac - - `$HOME/.ethereum` for Ubuntu - - `$GOPATH/src/gihub.com/8thlight/vulcanizedb/test_data_dir/geth.ipc` for private blockchain. - -**Note the location of the ipc file is outputted when you connect to geth. It is needed to for configuration** - -## Start Vulcanize DB -1. Start geth -2. In a separate terminal start vulcanize_db - - `vulcanize sync --config --starting-block-number ` - -## Watch contract events -1. Start geth -2. In a separate terminal start vulcanize_db - - `vulcanize sync --config --starting-block-number ` -3. Create event filter - - `vulcanize addFilter --config --filter-filepath ` - -### Configuring Additional Environments - -You can create configuration files for additional environments. - - * Among other things, it will require the IPC file path - * See `environments/private.toml` for an example - * You will need to do this if you want to run a node connecting to the public blockchain - -## Running the Tests - -### Unit Tests - -1. `go test ./pkg/...` - -### Integration Test - -In order to run the integration tests, you will need to run them against a real blockchain. At the moment the integration tests require [Geth v1.7.2](https://ethereum.github.io/go-ethereum/downloads/) as they depend on the `--dev` mode, which changed in v1.7.3 - -1. Run `./scripts/start_private_blockchain` as a separate process. -2. `go test ./...` to run all tests. diff --git a/db/schema.sql b/db/schema.sql index 9ba46900..eb98c994 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -404,6 +404,14 @@ ALTER TABLE ONLY watched_contracts ADD CONSTRAINT contract_hash_uc UNIQUE (contract_hash); +-- +-- Name: logs log_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY logs + ADD CONSTRAINT log_uc UNIQUE (block_number, index); + + -- -- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- diff --git a/commands/main.go b/main.go similarity index 54% rename from commands/main.go rename to main.go index 50e90e3b..8bdcc8e2 100644 --- a/commands/main.go +++ b/main.go @@ -1,7 +1,7 @@ package main import ( - "github.com/8thlight/vulcanizedb/commands/cmd" + "github.com/8thlight/vulcanizedb/cmd" ) func main() { From ed5a8f5bc4c7340a83902ff808b5ec7fe24361cd Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Thu, 25 Jan 2018 17:19:15 -0600 Subject: [PATCH 17/22] WIP --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ae38cd45..2865911e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# Vulcanize DB +# Vulcanize DB + +test change [![Build Status](https://travis-ci.com/vulcanize/vulcanizedb.svg?token=GKv2Y33qsFnfYgejjvYx&branch=master)](https://travis-ci.com/vulcanize/vulcanizedb) @@ -115,4 +117,4 @@ You can create configuration files for additional environments. In order to run the integration tests, you will need to run them against a real blockchain. At the moment the integration tests require [Geth v1.7.2](https://ethereum.github.io/go-ethereum/downloads/) as they depend on the `--dev` mode, which changed in v1.7.3 1. Run `./scripts/start_private_blockchain` as a separate process. -2. `go test ./...` to run all tests. \ No newline at end of file +2. `go test ./...` to run all tests. From afdeadeca1784b17931d7c2900fd0c5346c72de7 Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Thu, 25 Jan 2018 18:10:54 -0600 Subject: [PATCH 18/22] Update deps, schema and ci --- .travis.yml | 2 ++ Gopkg.lock | 70 ++------------------------------------------------- README.md | 2 +- db/schema.sql | 8 ------ 4 files changed, 5 insertions(+), 77 deletions(-) diff --git a/.travis.yml b/.travis.yml index c3c9b8d8..9cac17ce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,8 @@ services: addons: postgresql: "9.6" +go_import_path: github.com/vulcanize/vulcanizedb + before_install: # dep - go get -u github.com/golang/dep/cmd/dep diff --git a/Gopkg.lock b/Gopkg.lock index a7bbf097..2fd831a1 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -7,12 +7,6 @@ revision = "b26d9c308763d68093482582cea63d69be07a0f0" version = "v0.3.0" -[[projects]] - branch = "master" - name = "github.com/MichaelTJones/walk" - packages = ["."] - revision = "4748e29d5718c2df4028a6543edf86fd8cc0f881" - [[projects]] branch = "master" name = "github.com/aristanetworks/goarista" @@ -61,12 +55,6 @@ packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"] revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" -[[projects]] - branch = "master" - name = "github.com/howeyc/gopass" - packages = ["."] - revision = "bf9dde6d0d2c004a008c27aaee91170c786f6db8" - [[projects]] branch = "master" name = "github.com/huin/goupnp" @@ -103,42 +91,6 @@ revision = "d419a98cdbed11a922bf76f257b7c4be79b50e73" version = "v1.7.4" -[[projects]] - name = "github.com/mattn/go-colorable" - packages = ["."] - revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" - version = "v0.0.9" - -[[projects]] - name = "github.com/mattn/go-isatty" - packages = ["."] - revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" - version = "v0.0.3" - -[[projects]] - branch = "master" - name = "github.com/mgutz/ansi" - packages = ["."] - revision = "9520e82c474b0a04dd04f8a40959027271bab992" - -[[projects]] - branch = "master" - name = "github.com/mgutz/minimist" - packages = ["."] - revision = "39eb8cf573ca29344bd7d7e6ba4d7febdebd37a9" - -[[projects]] - name = "github.com/mgutz/str" - packages = ["."] - revision = "968bf66e3da857419e4f6e71b2d5c9ae95682dc4" - version = "v1.2.0" - -[[projects]] - name = "github.com/mgutz/to" - packages = ["."] - revision = "00c06406c2dd2e011f153a6502a21473676db33f" - version = "v1.0.0" - [[projects]] branch = "master" name = "github.com/mitchellh/go-homedir" @@ -151,12 +103,6 @@ packages = ["."] revision = "b4575eea38cca1123ec2dc90c26529b5c5acfcff" -[[projects]] - name = "github.com/nozzle/throttler" - packages = ["."] - revision = "d9b45f19996c645d38c9266d1f5cf1990e930119" - version = "v1.0" - [[projects]] name = "github.com/onsi/ginkgo" packages = [".","config","internal/codelocation","internal/containernode","internal/failer","internal/leafnodes","internal/remote","internal/spec","internal/spec_iterator","internal/specrunner","internal/suite","internal/testingtproxy","internal/writer","reporters","reporters/stenographer","reporters/stenographer/support/go-colorable","reporters/stenographer/support/go-isatty","types"] @@ -229,12 +175,6 @@ packages = ["leveldb","leveldb/cache","leveldb/comparer","leveldb/errors","leveldb/filter","leveldb/iterator","leveldb/journal","leveldb/memdb","leveldb/opt","leveldb/storage","leveldb/table","leveldb/util"] revision = "adf24ef3f94bd13ec4163060b21a5678f22b429b" -[[projects]] - branch = "master" - name = "golang.org/x/crypto" - packages = ["ssh/terminal"] - revision = "94eea52f7b742c7cbe0b03b22f0c4c8631ece122" - [[projects]] branch = "master" name = "golang.org/x/net" @@ -244,7 +184,7 @@ [[projects]] branch = "master" name = "golang.org/x/sys" - packages = ["unix","windows"] + packages = ["unix"] revision = "a0f4589a76f1f83070cb9e5613809e1d07b97c13" [[projects]] @@ -259,12 +199,6 @@ revision = "57907de300222151a123d29255ed17f5ed43fad3" version = "v0.1.0" -[[projects]] - name = "gopkg.in/godo.v2" - packages = [".","glob","util","watcher","watcher/fswatch"] - revision = "b5fd2f0bef1ebe832e628cfad18ab1cc707f65a1" - version = "v2.0.9" - [[projects]] branch = "v2" name = "gopkg.in/karalabe/cookiejar.v2" @@ -286,6 +220,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "46080c5bb453cb31ea731741d5defda225e2dc75bf7c11179d69422c82de37ca" + inputs-digest = "641a04f7f89572adf1ebd051d0839eb5b03fdc67bb50838bfac98832246636f0" solver-name = "gps-cdcl" solver-version = 1 diff --git a/README.md b/README.md index 7f0ea26a..9f4cdb4c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Vulcanize DB -[![Build Status](https://travis-ci.com/8thlight/vulcanizedb.svg?token=GKv2Y33qsFnfYgejjvYx&branch=master)](https://travis-ci.com/8thlight/vulcanizedb) +[![Build Status](https://travis-ci.com/8thlight/vulcanizedb.svg?token=3psFYN2533rYjhRbvjte&branch=master)](https://travis-ci.com/8thlight/vulcanizedb) ### Dependencies diff --git a/db/schema.sql b/db/schema.sql index eb98c994..9ba46900 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -404,14 +404,6 @@ ALTER TABLE ONLY watched_contracts ADD CONSTRAINT contract_hash_uc UNIQUE (contract_hash); --- --- Name: logs log_uc; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY logs - ADD CONSTRAINT log_uc UNIQUE (block_number, index); - - -- -- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- From 3b31e2a342044613961094242598505078bcfa41 Mon Sep 17 00:00:00 2001 From: Matt K <1036969+mkrump@users.noreply.github.com> Date: Fri, 26 Jan 2018 13:38:14 -0600 Subject: [PATCH 19/22] Update README, add State sync test (#1) --- Makefile | 11 +++++++++++ README.md | 35 ++++++++++++++++------------------- cmd/sync.go | 4 ++++ 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 43a43a5a..5e1469eb 100644 --- a/Makefile +++ b/Makefile @@ -38,3 +38,14 @@ build: dep test: $(GINKGO) ginkgo -r + +createprivate: + #!/bin/bash + echo "Deleting test blockchain" + rm -rf test_data_dir + echo "Creating test blockchain with a new account" + mkdir test_data_dir + geth --dev --datadir test_data_dir --password .private_blockchain_password account new + +startprivate: createprivate + geth --datadir test_data_dir --dev --nodiscover --mine --minerthreads 1 --maxpeers 0 --verbosity 3 --unlock 0 --password .private_blockchain_password --rpc diff --git a/README.md b/README.md index 9f4cdb4c..23a45d42 100644 --- a/README.md +++ b/README.md @@ -18,40 +18,32 @@ 2. Create a superuser for yourself and make sure `psql --list` works without prompting for a password. 3. `createdb vulcanize_private` 4. `cd $GOPATH/src/github.com/vulcanize/vulcanizedb` -5. Import the schema: - `psql vulcanize_private < db/schema.sql` - or run the migrations: - `make migrate HOST_NAME=localhost NAME=vulcanize_public PORT=5432` +5. Import the schema: `psql vulcanize_private < db/schema.sql` + + or run the migrations: `make migrate HOST_NAME=localhost NAME=vulcanize_public PORT=5432` * See below for configuring additional environments Adding a new migration: `./scripts/create_migration ` ## Start syncing with postgres -1. Start geth +1. Start geth node (**if fast syncing wait for geth to finsh initial sync**) 2. In a separate terminal start vulcanize_db - `vulcanizedb sync --config --starting-block-number ` * see `environments` for example config -## Watch specific contract events -1. Start geth +## Watch specific events +1. Start geth 2. In a separate terminal start vulcanize_db - `vulcanizedb sync --config --starting-block-number ` 3. Create event filter - `vulcanizedb addFilter --config --filter-filepath ` - - * see `filters` for example filter +4. The filters are tracked in the `log_filters` table and the filtered events +will show up in the `watched_log_events` view + * see `./filters` for example filter ## Development Setup -### Cloning the Repository (Private repo only) - -1. `git config --global url."git@github.com:".insteadOf "https://github.com/"` - - By default, `go get` does not work for private GitHub repos. This will fix that. -2. `go get github.com/vulcanize/vulcanizedb` -3. `cd $GOPATH/src/github.com/vulcanize/vulcanizedb` -4. `dep ensure` - ### Creating/Using a test node Syncing the against the public network takes many hours for the initial sync and will download 20+ GB of data. @@ -80,5 +72,10 @@ The default location for Ethereum is: In order to run the integration tests, you will need to run them against a real blockchain. At the moment the integration tests require [Geth v1.7.2](https://ethereum.github.io/go-ethereum/downloads/) as they depend on the `--dev` mode, which changed in v1.7.3 -1. Run `./scripts/start_private_blockchain` as a separate process. -2. `go test ./...` to run all tests. +1. Run `make startprivate` in a separate terminal +2. Setup a test database and import the schema: + + `createdb vulcanize_private` + + `psql vulcanize_private < db/schema.sql` +3. `go test ./...` to run all tests. diff --git a/cmd/sync.go b/cmd/sync.go index b8e742f4..748f23ce 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -11,6 +11,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/repositories" "github.com/vulcanize/vulcanizedb/utils" "github.com/spf13/cobra" + "log" ) // syncCmd represents the sync command @@ -58,6 +59,9 @@ func sync() { defer ticker.Stop() blockchain := geth.NewBlockchain(ipc) + if blockchain.LastBlock().Int64() == 0 { + log.Fatal("geth initial: state sync not finished") + } repository := utils.LoadPostgres(databaseConfig, blockchain.Node()) validator := history.NewBlockValidator(blockchain, repository, 15) From 662d7e1d794334dbdc5d97508c80105cf7b67315 Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Fri, 26 Jan 2018 13:51:13 -0600 Subject: [PATCH 20/22] Update README --- README.md | 55 +++++++++++++++++++++++-------------------------------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 23a45d42..c3201cc4 100644 --- a/README.md +++ b/README.md @@ -23,45 +23,34 @@ or run the migrations: `make migrate HOST_NAME=localhost NAME=vulcanize_public PORT=5432` * See below for configuring additional environments -Adding a new migration: `./scripts/create_migration ` - -## Start syncing with postgres -1. Start geth node (**if fast syncing wait for geth to finsh initial sync**) -2. In a separate terminal start vulcanize_db - - `vulcanizedb sync --config --starting-block-number ` - - * see `environments` for example config - -## Watch specific events -1. Start geth -2. In a separate terminal start vulcanize_db - - `vulcanizedb sync --config --starting-block-number ` -3. Create event filter - - `vulcanizedb addFilter --config --filter-filepath ` -4. The filters are tracked in the `log_filters` table and the filtered events -will show up in the `watched_log_events` view - * see `./filters` for example filter - -## Development Setup - -### Creating/Using a test node - -Syncing the against the public network takes many hours for the initial sync and will download 20+ GB of data. -Here are some instructions for creating a private test node that does not depend on having a network connection. - -1. Run `./scripts/setup` to create a private blockchain with a new account. - * This will result in a warning. -2. Run `./scripts/start_private_blockchain`. - ### IPC File Paths The default location for Ethereum is: - `$HOME/Library/Ethereum` for Mac - `$HOME/.ethereum` for Ubuntu - - `$GOPATH/src/gihub.com/vulcanize/vulcanizedb/test_data_dir/geth.ipc` for private blockchain. + - `$GOPATH/src/gihub.com/vulcanize/vulcanizedb/test_data_dir/geth.ipc` for private node. **Note the location of the ipc file is printed to the console when you start geth. It is needed to for configuration** +## Start syncing with postgres +1. Start geth node (**if fast syncing wait for geth to finsh initial sync**) +2. In a separate terminal start vulcanize_db + - `vulcanizedb sync --config --starting-block-number ` + + * see `./environments` for example config + +## Watch specific events +1. Start geth +2. In a separate terminal start vulcanize_db + - `vulcanizedb sync --config --starting-block-number ` +3. Create event filter + - `vulcanizedb addFilter --config --filter-filepath ` + * see `./filters` for example filter +4. The filters are tracked in the `log_filters` table and the filtered events +will show up in the `watched_log_events` view + +## Development Setup + ## Running the Tests ### Unit Tests @@ -70,7 +59,9 @@ The default location for Ethereum is: ### Integration Test -In order to run the integration tests, you will need to run them against a real blockchain. At the moment the integration tests require [Geth v1.7.2](https://ethereum.github.io/go-ethereum/downloads/) as they depend on the `--dev` mode, which changed in v1.7.3 +### Creating/Using a test node + +In order to run the integration tests, you will need to run them against a real node. At the moment the integration tests require [Geth v1.7.2](https://ethereum.github.io/go-ethereum/downloads/) as they depend on the `--dev` mode, which changed in v1.7.3 1. Run `make startprivate` in a separate terminal 2. Setup a test database and import the schema: From 69545b72f000053decfdc953d4609d8d311dda6f Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Fri, 26 Jan 2018 13:52:45 -0600 Subject: [PATCH 21/22] Update README --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index c3201cc4..20c77aa1 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,6 @@ The default location for Ethereum is: 4. The filters are tracked in the `log_filters` table and the filtered events will show up in the `watched_log_events` view -## Development Setup - ## Running the Tests ### Unit Tests From 0800e331d59beb9b6491ff552e4a843e228f6dc0 Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Fri, 26 Jan 2018 13:53:26 -0600 Subject: [PATCH 22/22] Update README --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 20c77aa1..7a78ccce 100644 --- a/README.md +++ b/README.md @@ -57,8 +57,6 @@ will show up in the `watched_log_events` view ### Integration Test -### Creating/Using a test node - In order to run the integration tests, you will need to run them against a real node. At the moment the integration tests require [Geth v1.7.2](https://ethereum.github.io/go-ethereum/downloads/) as they depend on the `--dev` mode, which changed in v1.7.3 1. Run `make startprivate` in a separate terminal