From 431be46005237aa73a26489b565554759824cfc3 Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Mon, 15 Jan 2018 10:52:50 -0600 Subject: [PATCH 1/7] Add store logs with receipts --- pkg/repositories/in_memory.go | 1 + pkg/repositories/postgres.go | 3 ++ pkg/repositories/testing/helpers.go | 81 +++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) diff --git a/pkg/repositories/in_memory.go b/pkg/repositories/in_memory.go index 58cf4fe1..0a609231 100644 --- a/pkg/repositories/in_memory.go +++ b/pkg/repositories/in_memory.go @@ -102,6 +102,7 @@ func (repository *InMemory) CreateOrUpdateBlock(block core.Block) error { repository.blocks[block.Number] = block for _, transaction := range block.Transactions { repository.receipts[transaction.Hash] = transaction.Receipt + repository.logs[transaction.TxHash] = transaction.Logs } return nil } diff --git a/pkg/repositories/postgres.go b/pkg/repositories/postgres.go index 4418073a..d7d82402 100644 --- a/pkg/repositories/postgres.go +++ b/pkg/repositories/postgres.go @@ -344,6 +344,9 @@ func (repository Postgres) createTransaction(tx *sql.Tx, blockId int64, transact if err != nil { return err } + if len(transaction.Receipt.Logs) > 0 { + err = repository.CreateLogs(transaction.Receipt.Logs) + } } return nil } diff --git a/pkg/repositories/testing/helpers.go b/pkg/repositories/testing/helpers.go index f2b42138..2ce90b3d 100644 --- a/pkg/repositories/testing/helpers.go +++ b/pkg/repositories/testing/helpers.go @@ -504,6 +504,87 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. {blockNumber: 1, Index: 1}}, )) }) + + It("saves the logs attached to a receipt", func() { + logs := []core.Log{{ + Address: "0x8a4774fe82c63484afef97ca8d89a6ea5e21f973", + BlockNumber: 4745407, + Data: "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000645a68669900000000000000000000000000000000000000000000003397684ab5869b0000000000000000000000000000000000000000000000000000000000005a36053200000000000000000000000099041f808d598b782d5a3e498681c2452a31da08", + Index: 86, + Topics: map[int]string{ + 0: "0x5a68669900000000000000000000000000000000000000000000000000000000", + 1: "0x000000000000000000000000d0148dad63f73ce6f1b6c607e3413dcf1ff5f030", + 2: "0x00000000000000000000000000000000000000000000003397684ab5869b0000", + 3: "0x000000000000000000000000000000000000000000000000000000005a360532", + }, + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + }, { + Address: "0x99041f808d598b782d5a3e498681c2452a31da08", + BlockNumber: 4745407, + Data: "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000418178358", + Index: 87, + Topics: map[int]string{ + 0: "0x1817835800000000000000000000000000000000000000000000000000000000", + 1: "0x0000000000000000000000008a4774fe82c63484afef97ca8d89a6ea5e21f973", + 2: "0x0000000000000000000000000000000000000000000000000000000000000000", + 3: "0x0000000000000000000000000000000000000000000000000000000000000000", + }, + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + }, { + Address: "0x99041f808d598b782d5a3e498681c2452a31da08", + BlockNumber: 4745407, + Data: "0x00000000000000000000000000000000000000000000003338f64c8423af4000", + Index: 88, + Topics: map[int]string{ + 0: "0x296ba4ca62c6c21c95e828080cb8aec7481b71390585605300a8a76f9e95b527", + }, + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + }, + } + receipt := core.Receipt{ + ContractAddress: "", + CumulativeGasUsed: 7481414, + GasUsed: 60711, + Logs: logs, + Bloom: "0x00000800000000000000001000000000000000400000000080000000000000000000400000010000000000000000000000000000040000000000000002000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000800004000000000000001000000000000000000000000000002000000480000000000000002000000000000000020000000000000000000000000000000000000000080000000000180000c00000000000000002000002000000040000000000000000000000000000010000000000020000000000000000000002000000000000000000000000400800000000000000000", + Status: 1, + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + } + transaction := + core.Transaction{ + Hash: receipt.TxHash, + Receipt: receipt, + } + + block := core.Block{Transactions: []core.Transaction{transaction}} + repository.CreateOrUpdateBlock(block) + retrievedLogs := repository.FindLogs("0x99041f808d598b782d5a3e498681c2452a31da08", 4745407) + + Expect(retrievedLogs).To(Not(BeZero())) + }) + + It("saves the logs attached to a receipt", func() { + logs := make([]core.Log, 0) + receipt := core.Receipt{ + Logs: logs, + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + } + transaction := core.Transaction{ + Hash: receipt.TxHash, + Receipt: receipt, + } + + block := core.Block{ + Transactions: []core.Transaction{transaction}, + } + repository.CreateOrUpdateBlock(block) + + retrievedLogs := repository.FindLogs("0x99041f808d598b782d5a3e498681c2452a31da08", 4745407) + _, err := repository.FindReceipt(receipt.TxHash) + + Expect(err).To(Not(HaveOccurred())) + Expect(retrievedLogs).To(BeZero()) + }) }) Describe("Saving receipts", func() { From 50f00b80c1bd978a1f23fe92861eef0d73087d52 Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Mon, 15 Jan 2018 11:21:08 -0600 Subject: [PATCH 2/7] Lowercase log address --- pkg/geth/log_to_core_log.go | 4 +++- pkg/geth/log_to_core_log_test.go | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/geth/log_to_core_log.go b/pkg/geth/log_to_core_log.go index f40983f4..bce6fd4c 100644 --- a/pkg/geth/log_to_core_log.go +++ b/pkg/geth/log_to_core_log.go @@ -1,6 +1,8 @@ package geth import ( + "strings" + "github.com/8thlight/vulcanizedb/pkg/core" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -13,7 +15,7 @@ func LogToCoreLog(gethLog types.Log) core.Log { hexTopics[i] = topic.Hex() } return core.Log{ - Address: gethLog.Address.Hex(), + Address: strings.ToLower(gethLog.Address.Hex()), BlockNumber: int64(gethLog.BlockNumber), Topics: hexTopics, diff --git a/pkg/geth/log_to_core_log_test.go b/pkg/geth/log_to_core_log_test.go index 44448ce3..ad15e826 100644 --- a/pkg/geth/log_to_core_log_test.go +++ b/pkg/geth/log_to_core_log_test.go @@ -1,6 +1,8 @@ package geth_test import ( + "strings" + "github.com/8thlight/vulcanizedb/pkg/core" "github.com/8thlight/vulcanizedb/pkg/geth" "github.com/ethereum/go-ethereum/common" @@ -14,7 +16,7 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { It("converts geth log to internal log format", func() { gethLog := types.Log{ - Address: common.HexToAddress("0xecf8f87f810ecf450940c9f60066b4a7a501d6a7"), + Address: common.HexToAddress("0x448a5065aeBB8E423F0896E6c5D525C040f59af3"), BlockHash: common.HexToHash("0x656c34545f90a730a19008c0e7a7cd4fb3895064b48d6d69761bd5abad681056"), BlockNumber: 2019236, Data: hexutil.MustDecode("0x000000000000000000000000000000000000000000000001a055690d9db80000"), @@ -28,7 +30,7 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { } expected := core.Log{ - Address: gethLog.Address.Hex(), + Address: strings.ToLower(gethLog.Address.Hex()), BlockNumber: int64(gethLog.BlockNumber), Data: hexutil.Encode(gethLog.Data), TxHash: gethLog.TxHash.Hex(), From 9ee13e715dbe2b427032b5ce128ccd621a0fa647 Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Mon, 15 Jan 2018 14:44:47 -0600 Subject: [PATCH 3/7] Add topics type --- integration_test/contract_test.go | 4 +- pkg/core/log.go | 7 +-- pkg/core/topics.go | 3 ++ pkg/geth/blockchain.go | 2 +- pkg/geth/log_to_core_log.go | 31 ++++++++----- pkg/geth/log_to_core_log_test.go | 7 ++- pkg/repositories/postgres.go | 69 +++++++++++++++-------------- pkg/repositories/testing/helpers.go | 21 ++++----- 8 files changed, 80 insertions(+), 64 deletions(-) create mode 100644 pkg/core/topics.go diff --git a/integration_test/contract_test.go b/integration_test/contract_test.go index 6d0af83f..33784973 100644 --- a/integration_test/contract_test.go +++ b/integration_test/contract_test.go @@ -107,8 +107,8 @@ var _ = Describe("Reading contracts", func() { expectedLogZero := core.Log{ BlockNumber: 4703824, TxHash: "0xf896bfd1eb539d881a1a31102b78de9f25cd591bf1fe1924b86148c0b205fd5d", - Address: "0xd26114cd6EE289AccF82350c8d8487fedB8A0C07", - Topics: map[int]string{ + Address: "0xd26114cd6ee289accf82350c8d8487fedb8a0c07", + Topics: core.Topics{ 0: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", 1: "0x000000000000000000000000fbb1b73c4f0bda4f67dca266ce6ef42f520fbb98", 2: "0x000000000000000000000000d26114cd6ee289accf82350c8d8487fedb8a0c07", diff --git a/pkg/core/log.go b/pkg/core/log.go index 9688b46c..c96817f9 100644 --- a/pkg/core/log.go +++ b/pkg/core/log.go @@ -4,7 +4,8 @@ type Log struct { BlockNumber int64 TxHash string Address string - Topics map[int]string - Index int64 - Data string + Topics + Index int64 + Data string + Removed bool } diff --git a/pkg/core/topics.go b/pkg/core/topics.go new file mode 100644 index 00000000..2f1acb9d --- /dev/null +++ b/pkg/core/topics.go @@ -0,0 +1,3 @@ +package core + +type Topics [4]string diff --git a/pkg/geth/blockchain.go b/pkg/geth/blockchain.go index c519706f..aad367d3 100644 --- a/pkg/geth/blockchain.go +++ b/pkg/geth/blockchain.go @@ -57,7 +57,7 @@ func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumbe if err != nil { return []core.Log{}, err } - logs := GethLogsToCoreLogs(gethLogs) + logs := LogsToCoreLogs(gethLogs) return logs, nil } diff --git a/pkg/geth/log_to_core_log.go b/pkg/geth/log_to_core_log.go index bce6fd4c..f3bc3534 100644 --- a/pkg/geth/log_to_core_log.go +++ b/pkg/geth/log_to_core_log.go @@ -4,16 +4,31 @@ import ( "strings" "github.com/8thlight/vulcanizedb/pkg/core" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" ) -func LogToCoreLog(gethLog types.Log) core.Log { - topics := gethLog.Topics - var hexTopics = make(map[int]string) +func LogsToCoreLogs(gethLogs []types.Log) []core.Log { + var logs []core.Log + for _, log := range gethLogs { + log := LogToCoreLog(log) + logs = append(logs, log) + } + return logs +} + +func MakeTopics(topics []common.Hash) core.Topics { + var hexTopics core.Topics for i, topic := range topics { hexTopics[i] = topic.Hex() } + return hexTopics +} + +func LogToCoreLog(gethLog types.Log) core.Log { + topics := gethLog.Topics + hexTopics := MakeTopics(topics) return core.Log{ Address: strings.ToLower(gethLog.Address.Hex()), @@ -22,14 +37,6 @@ func LogToCoreLog(gethLog types.Log) core.Log { TxHash: gethLog.TxHash.Hex(), Index: int64(gethLog.Index), Data: hexutil.Encode(gethLog.Data), + Removed: gethLog.Removed, } } - -func GethLogsToCoreLogs(gethLogs []types.Log) []core.Log { - var logs []core.Log - for _, log := range gethLogs { - log := LogToCoreLog(log) - logs = append(logs, log) - } - return logs -} diff --git a/pkg/geth/log_to_core_log_test.go b/pkg/geth/log_to_core_log_test.go index ad15e826..a470b91c 100644 --- a/pkg/geth/log_to_core_log_test.go +++ b/pkg/geth/log_to_core_log_test.go @@ -27,6 +27,7 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"), common.HexToHash("0x00000000000000000000000080b2c9d7cbbf30a1b0fc8983c647d754c6525615"), }, + Removed: true, } expected := core.Log{ @@ -35,10 +36,11 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { Data: hexutil.Encode(gethLog.Data), TxHash: gethLog.TxHash.Hex(), Index: 2, - Topics: map[int]string{ + Topics: core.Topics{ 0: common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef").Hex(), 1: common.HexToHash("0x00000000000000000000000080b2c9d7cbbf30a1b0fc8983c647d754c6525615").Hex(), }, + Removed: gethLog.Removed, } coreLog := geth.LogToCoreLog(gethLog) @@ -46,6 +48,7 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { Expect(coreLog.Address).To(Equal(expected.Address)) Expect(coreLog.BlockNumber).To(Equal(expected.BlockNumber)) Expect(coreLog.Data).To(Equal(expected.Data)) + Expect(coreLog.Removed).To(Equal(expected.Removed)) Expect(coreLog.Index).To(Equal(expected.Index)) Expect(coreLog.Topics[0]).To(Equal(expected.Topics[0])) Expect(coreLog.Topics[1]).To(Equal(expected.Topics[1])) @@ -84,7 +87,7 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { expectedOne := geth.LogToCoreLog(gethLogOne) expectedTwo := geth.LogToCoreLog(gethLogTwo) - coreLogs := geth.GethLogsToCoreLogs([]types.Log{gethLogOne, gethLogTwo}) + coreLogs := geth.LogsToCoreLogs([]types.Log{gethLogOne, gethLogTwo}) Expect(len(coreLogs)).To(Equal(2)) Expect(coreLogs[0]).To(Equal(expectedOne)) diff --git a/pkg/repositories/postgres.go b/pkg/repositories/postgres.go index d7d82402..5461fe53 100644 --- a/pkg/repositories/postgres.go +++ b/pkg/repositories/postgres.go @@ -62,35 +62,6 @@ func (repository Postgres) SetBlocksStatus(chainHead int64) { cutoff) } -func (repository Postgres) CreateLogs(logs []core.Log) error { - tx, _ := repository.Db.BeginTx(context.Background(), nil) - for _, tlog := range logs { - _, err := tx.Exec( - `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ON CONFLICT (index, block_number) - DO UPDATE - SET block_number = $1, - address = $2, - tx_hash = $3, - index = $4, - topic0 = $5, - topic1 = $6, - topic2 = $7, - topic3 = $8, - data = $9 - `, - tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, - ) - if err != nil { - tx.Rollback() - return ErrDBInsertFailed - } - } - tx.Commit() - return nil -} - func (repository Postgres) FindLogs(address string, blockNumber int64) []core.Log { logRows, _ := repository.Db.Query( `SELECT block_number, @@ -353,17 +324,48 @@ func (repository Postgres) createTransaction(tx *sql.Tx, blockId int64, transact func (repository Postgres) createReceipt(tx *sql.Tx, transactionId int, receipt core.Receipt) error { //Not currently persisting log bloom filters - _, err := tx.Exec( + var receiptId int + err := tx.QueryRow( `INSERT INTO receipts (contract_address, tx_hash, cumulative_gas_used, gas_used, state_root, status, transaction_id) - VALUES ($1, $2, $3, $4, $5, $6, $7)`, - receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, transactionId) + VALUES ($1, $2, $3, $4, $5, $6, $7) + RETURNING id`, + receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, transactionId).Scan(&receiptId) if err != nil { return err } return nil } +func (repository Postgres) CreateLogs(logs []core.Log) error { + tx, _ := repository.Db.BeginTx(context.Background(), nil) + for _, tlog := range logs { + _, err := tx.Exec( + `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (index, block_number) + DO UPDATE + SET block_number = $1, + address = $2, + tx_hash = $3, + index = $4, + topic0 = $5, + topic1 = $6, + topic2 = $7, + topic3 = $8, + data = $9 + `, + tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, + ) + if err != nil { + tx.Rollback() + return ErrDBInsertFailed + } + } + tx.Commit() + return nil +} + func loadReceipt(receiptsRow *sql.Row) (core.Receipt, error) { var contractAddress string var txHash string @@ -445,7 +447,7 @@ func (repository Postgres) loadLogs(logsRows *sql.Rows) []core.Log { var txHash string var index int64 var data string - topics := make([]string, 4) + var topics core.Topics logsRows.Scan(&blockNumber, &address, &txHash, &index, &topics[0], &topics[1], &topics[2], &topics[3], &data) log := core.Log{ BlockNumber: blockNumber, @@ -454,7 +456,6 @@ func (repository Postgres) loadLogs(logsRows *sql.Rows) []core.Log { Index: index, Data: data, } - log.Topics = make(map[int]string) for i, topic := range topics { log.Topics[i] = topic } diff --git a/pkg/repositories/testing/helpers.go b/pkg/repositories/testing/helpers.go index 2ce90b3d..251d043d 100644 --- a/pkg/repositories/testing/helpers.go +++ b/pkg/repositories/testing/helpers.go @@ -399,7 +399,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, Data: "xabc", }}, ) @@ -428,7 +428,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, Data: "xABC", }, }) @@ -437,7 +437,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, Data: "xXYZ", }, }) @@ -452,7 +452,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, Data: "xabc", }}, ) @@ -461,7 +461,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 1, Address: "x123", TxHash: "x789", - Topics: map[int]string{0: "x111", 1: "x222", 2: "x333"}, + Topics: [4]string{0: "x111", 1: "x222", 2: "x333"}, Data: "xdef", }}, ) @@ -470,7 +470,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, Data: "xabc", }}, ) @@ -511,7 +511,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. BlockNumber: 4745407, Data: "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000645a68669900000000000000000000000000000000000000000000003397684ab5869b0000000000000000000000000000000000000000000000000000000000005a36053200000000000000000000000099041f808d598b782d5a3e498681c2452a31da08", Index: 86, - Topics: map[int]string{ + Topics: core.Topics{ 0: "0x5a68669900000000000000000000000000000000000000000000000000000000", 1: "0x000000000000000000000000d0148dad63f73ce6f1b6c607e3413dcf1ff5f030", 2: "0x00000000000000000000000000000000000000000000003397684ab5869b0000", @@ -523,7 +523,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. BlockNumber: 4745407, Data: "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000418178358", Index: 87, - Topics: map[int]string{ + Topics: core.Topics{ 0: "0x1817835800000000000000000000000000000000000000000000000000000000", 1: "0x0000000000000000000000008a4774fe82c63484afef97ca8d89a6ea5e21f973", 2: "0x0000000000000000000000000000000000000000000000000000000000000000", @@ -535,7 +535,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. BlockNumber: 4745407, Data: "0x00000000000000000000000000000000000000000000003338f64c8423af4000", Index: 88, - Topics: map[int]string{ + Topics: core.Topics{ 0: "0x296ba4ca62c6c21c95e828080cb8aec7481b71390585605300a8a76f9e95b527", }, TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", @@ -557,7 +557,8 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. } block := core.Block{Transactions: []core.Transaction{transaction}} - repository.CreateOrUpdateBlock(block) + err := repository.CreateOrUpdateBlock(block) + Expect(err).To(Not(HaveOccurred())) retrievedLogs := repository.FindLogs("0x99041f808d598b782d5a3e498681c2452a31da08", 4745407) Expect(retrievedLogs).To(Not(BeZero())) From 82c39a2c1febeaf4b64590cc5741af9292006570 Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Mon, 15 Jan 2018 15:27:45 -0600 Subject: [PATCH 4/7] Add fk constraint on logs --- .../1516050071_add_log_fk_constraint.down.sql | 9 ++++ .../1516050071_add_log_fk_constraint.up.sql | 12 +++++ pkg/core/log.go | 5 +- pkg/geth/blockchain.go | 9 +++- pkg/geth/log_to_core_log.go | 11 ++--- pkg/geth/log_to_core_log_test.go | 15 +++--- pkg/geth/receipt_to_core_receipt.go | 2 +- pkg/repositories/postgres.go | 49 ++++++++++++++++--- pkg/repositories/testing/helpers.go | 21 ++++---- 9 files changed, 94 insertions(+), 39 deletions(-) create mode 100644 db/migrations/1516050071_add_log_fk_constraint.down.sql create mode 100644 db/migrations/1516050071_add_log_fk_constraint.up.sql diff --git a/db/migrations/1516050071_add_log_fk_constraint.down.sql b/db/migrations/1516050071_add_log_fk_constraint.down.sql new file mode 100644 index 00000000..6ce8446f --- /dev/null +++ b/db/migrations/1516050071_add_log_fk_constraint.down.sql @@ -0,0 +1,9 @@ +BEGIN; + +ALTER TABLE logs + DROP CONSTRAINT receipts_fk; + +ALTER TABLE logs + DROP COLUMN receipt_id; + +COMMIT; \ No newline at end of file diff --git a/db/migrations/1516050071_add_log_fk_constraint.up.sql b/db/migrations/1516050071_add_log_fk_constraint.up.sql new file mode 100644 index 00000000..a0150d1a --- /dev/null +++ b/db/migrations/1516050071_add_log_fk_constraint.up.sql @@ -0,0 +1,12 @@ +BEGIN; + +ALTER TABLE logs + ADD COLUMN receipt_id INT; + +ALTER TABLE logs + ADD CONSTRAINT receipts_fk +FOREIGN KEY (receipt_id) +REFERENCES receipts (id) +ON DELETE CASCADE; + +COMMIT; \ No newline at end of file diff --git a/pkg/core/log.go b/pkg/core/log.go index c96817f9..984e0432 100644 --- a/pkg/core/log.go +++ b/pkg/core/log.go @@ -5,7 +5,6 @@ type Log struct { TxHash string Address string Topics - Index int64 - Data string - Removed bool + Index int64 + Data string } diff --git a/pkg/geth/blockchain.go b/pkg/geth/blockchain.go index aad367d3..5ff500c2 100644 --- a/pkg/geth/blockchain.go +++ b/pkg/geth/blockchain.go @@ -5,6 +5,8 @@ import ( "strings" + "log" + "github.com/8thlight/vulcanizedb/pkg/core" "github.com/8thlight/vulcanizedb/pkg/geth/node" "github.com/ethereum/go-ethereum" @@ -25,7 +27,10 @@ type Blockchain struct { func NewBlockchain(ipcPath string) *Blockchain { blockchain := Blockchain{} - rpcClient, _ := rpc.Dial(ipcPath) + rpcClient, err := rpc.Dial(ipcPath) + if err != nil { + log.Fatal(err) + } client := ethclient.NewClient(rpcClient) blockchain.node = node.Info(rpcClient) if infura := isInfuraNode(ipcPath); infura { @@ -57,7 +62,7 @@ func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumbe if err != nil { return []core.Log{}, err } - logs := LogsToCoreLogs(gethLogs) + logs := ToCoreLogs(gethLogs) return logs, nil } diff --git a/pkg/geth/log_to_core_log.go b/pkg/geth/log_to_core_log.go index f3bc3534..7b24175f 100644 --- a/pkg/geth/log_to_core_log.go +++ b/pkg/geth/log_to_core_log.go @@ -9,16 +9,16 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) -func LogsToCoreLogs(gethLogs []types.Log) []core.Log { +func ToCoreLogs(gethLogs []types.Log) []core.Log { var logs []core.Log for _, log := range gethLogs { - log := LogToCoreLog(log) + log := ToCoreLog(log) logs = append(logs, log) } return logs } -func MakeTopics(topics []common.Hash) core.Topics { +func makeTopics(topics []common.Hash) core.Topics { var hexTopics core.Topics for i, topic := range topics { hexTopics[i] = topic.Hex() @@ -26,9 +26,9 @@ func MakeTopics(topics []common.Hash) core.Topics { return hexTopics } -func LogToCoreLog(gethLog types.Log) core.Log { +func ToCoreLog(gethLog types.Log) core.Log { topics := gethLog.Topics - hexTopics := MakeTopics(topics) + hexTopics := makeTopics(topics) return core.Log{ Address: strings.ToLower(gethLog.Address.Hex()), @@ -37,6 +37,5 @@ func LogToCoreLog(gethLog types.Log) core.Log { TxHash: gethLog.TxHash.Hex(), Index: int64(gethLog.Index), Data: hexutil.Encode(gethLog.Data), - Removed: gethLog.Removed, } } diff --git a/pkg/geth/log_to_core_log_test.go b/pkg/geth/log_to_core_log_test.go index a470b91c..000276ff 100644 --- a/pkg/geth/log_to_core_log_test.go +++ b/pkg/geth/log_to_core_log_test.go @@ -27,7 +27,6 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"), common.HexToHash("0x00000000000000000000000080b2c9d7cbbf30a1b0fc8983c647d754c6525615"), }, - Removed: true, } expected := core.Log{ @@ -37,18 +36,16 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { TxHash: gethLog.TxHash.Hex(), Index: 2, Topics: core.Topics{ - 0: common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef").Hex(), - 1: common.HexToHash("0x00000000000000000000000080b2c9d7cbbf30a1b0fc8983c647d754c6525615").Hex(), + gethLog.Topics[0].Hex(), + gethLog.Topics[1].Hex(), }, - Removed: gethLog.Removed, } - coreLog := geth.LogToCoreLog(gethLog) + coreLog := geth.ToCoreLog(gethLog) Expect(coreLog.Address).To(Equal(expected.Address)) Expect(coreLog.BlockNumber).To(Equal(expected.BlockNumber)) Expect(coreLog.Data).To(Equal(expected.Data)) - Expect(coreLog.Removed).To(Equal(expected.Removed)) Expect(coreLog.Index).To(Equal(expected.Index)) Expect(coreLog.Topics[0]).To(Equal(expected.Topics[0])) Expect(coreLog.Topics[1]).To(Equal(expected.Topics[1])) @@ -84,10 +81,10 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { }, } - expectedOne := geth.LogToCoreLog(gethLogOne) - expectedTwo := geth.LogToCoreLog(gethLogTwo) + expectedOne := geth.ToCoreLog(gethLogOne) + expectedTwo := geth.ToCoreLog(gethLogTwo) - coreLogs := geth.LogsToCoreLogs([]types.Log{gethLogOne, gethLogTwo}) + coreLogs := geth.ToCoreLogs([]types.Log{gethLogOne, gethLogTwo}) Expect(len(coreLogs)).To(Equal(2)) Expect(coreLogs[0]).To(Equal(expectedOne)) diff --git a/pkg/geth/receipt_to_core_receipt.go b/pkg/geth/receipt_to_core_receipt.go index b9130f4b..7117a6b7 100644 --- a/pkg/geth/receipt_to_core_receipt.go +++ b/pkg/geth/receipt_to_core_receipt.go @@ -49,7 +49,7 @@ func setContractAddress(gethReceipt *types.Receipt) string { func dereferenceLogs(gethReceipt *types.Receipt) []core.Log { logs := []core.Log{} for _, log := range gethReceipt.Logs { - logs = append(logs, LogToCoreLog(*log)) + logs = append(logs, ToCoreLog(*log)) } return logs } diff --git a/pkg/repositories/postgres.go b/pkg/repositories/postgres.go index 5461fe53..ef63d396 100644 --- a/pkg/repositories/postgres.go +++ b/pkg/repositories/postgres.go @@ -310,19 +310,30 @@ func (repository Postgres) createTransaction(tx *sql.Tx, blockId int64, transact if err != nil { return err } - if transaction.Receipt.TxHash != "" { - err = repository.createReceipt(tx, transactionId, transaction.Receipt) + if hasReceipt(transaction) { + receiptId, err := repository.createReceipt(tx, transactionId, transaction.Receipt) if err != nil { return err } - if len(transaction.Receipt.Logs) > 0 { - err = repository.CreateLogs(transaction.Receipt.Logs) + if hasLogs(transaction) { + err = repository.createLogs(tx, transaction.Receipt.Logs, receiptId) + if err != nil { + return err + } } } return nil } -func (repository Postgres) createReceipt(tx *sql.Tx, transactionId int, receipt core.Receipt) error { +func hasLogs(transaction core.Transaction) bool { + return len(transaction.Receipt.Logs) > 0 +} + +func hasReceipt(transaction core.Transaction) bool { + return transaction.Receipt.TxHash != "" +} + +func (repository Postgres) createReceipt(tx *sql.Tx, transactionId int, receipt core.Receipt) (int, error) { //Not currently persisting log bloom filters var receiptId int err := tx.QueryRow( @@ -332,7 +343,33 @@ func (repository Postgres) createReceipt(tx *sql.Tx, transactionId int, receipt RETURNING id`, receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, transactionId).Scan(&receiptId) if err != nil { - return err + return receiptId, err + } + return receiptId, nil +} + +func (repository Postgres) createLogs(tx *sql.Tx, logs []core.Log, receiptId int) error { + for _, tlog := range logs { + _, err := tx.Exec( + `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (index, block_number) + DO UPDATE + SET block_number = $1, + address = $2, + tx_hash = $3, + index = $4, + topic0 = $5, + topic1 = $6, + topic2 = $7, + topic3 = $8, + data = $9 + `, + tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, + ) + if err != nil { + return ErrDBInsertFailed + } } return nil } diff --git a/pkg/repositories/testing/helpers.go b/pkg/repositories/testing/helpers.go index 251d043d..6a89fd29 100644 --- a/pkg/repositories/testing/helpers.go +++ b/pkg/repositories/testing/helpers.go @@ -399,7 +399,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, Data: "xabc", }}, ) @@ -428,7 +428,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, Data: "xABC", }, }) @@ -437,7 +437,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, Data: "xXYZ", }, }) @@ -452,7 +452,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, Data: "xabc", }}, ) @@ -461,7 +461,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 1, Address: "x123", TxHash: "x789", - Topics: [4]string{0: "x111", 1: "x222", 2: "x333"}, + Topics: core.Topics{0: "x111", 1: "x222", 2: "x333"}, Data: "xdef", }}, ) @@ -470,7 +470,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: [4]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, Data: "xabc", }}, ) @@ -561,13 +561,12 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Expect(err).To(Not(HaveOccurred())) retrievedLogs := repository.FindLogs("0x99041f808d598b782d5a3e498681c2452a31da08", 4745407) - Expect(retrievedLogs).To(Not(BeZero())) + expected := logs[1:] + Expect(retrievedLogs).To(Equal(expected)) }) - It("saves the logs attached to a receipt", func() { - logs := make([]core.Log, 0) + It("still saves receipts without logs", func() { receipt := core.Receipt{ - Logs: logs, TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", } transaction := core.Transaction{ @@ -580,11 +579,9 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. } repository.CreateOrUpdateBlock(block) - retrievedLogs := repository.FindLogs("0x99041f808d598b782d5a3e498681c2452a31da08", 4745407) _, err := repository.FindReceipt(receipt.TxHash) Expect(err).To(Not(HaveOccurred())) - Expect(retrievedLogs).To(BeZero()) }) }) From 08993cc6a4b43a2ccccc0397499fda7df0b98131 Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Tue, 16 Jan 2018 09:11:23 -0600 Subject: [PATCH 5/7] Add back schema dump --- Gododir/main.go | 4 + db/schema.sql | 476 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 480 insertions(+) create mode 100644 db/schema.sql diff --git a/Gododir/main.go b/Gododir/main.go index 09b49130..454e46e0 100644 --- a/Gododir/main.go +++ b/Gododir/main.go @@ -79,7 +79,9 @@ func tasks(p *do.Project) { cfg := cmd.LoadConfig(environment) connectString := config.DbConnectionString(cfg.Database) migrate := fmt.Sprintf("migrate -database '%s' -path ./db/migrations up", connectString) + dumpSchema := fmt.Sprintf("pg_dump -O -s %s > db/schema.sql", cfg.Database.Name) context.Bash(migrate) + context.Bash(dumpSchema) }) p.Task("rollback", nil, func(context *do.Context) { @@ -87,7 +89,9 @@ func tasks(p *do.Project) { cfg := cmd.LoadConfig(environment) connectString := config.DbConnectionString(cfg.Database) migrate := fmt.Sprintf("migrate -database '%s' -path ./db/migrations down 1", connectString) + dumpSchema := fmt.Sprintf("pg_dump -O -s %s > db/schema.sql", cfg.Database.Name) context.Bash(migrate) + context.Bash(dumpSchema) }) p.Task("showContractSummary", nil, func(context *do.Context) { diff --git a/db/schema.sql b/db/schema.sql new file mode 100644 index 00000000..0b7f6190 --- /dev/null +++ b/db/schema.sql @@ -0,0 +1,476 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 10.1 +-- Dumped by pg_dump version 10.1 + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; + + +-- +-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; + + +SET search_path = public, pg_catalog; + +SET default_tablespace = ''; + +SET default_with_oids = false; + +-- +-- Name: blocks; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE blocks ( + block_number bigint, + block_gaslimit double precision, + block_gasused double precision, + block_time double precision, + id integer NOT NULL, + block_difficulty bigint, + block_hash character varying(66), + block_nonce character varying(20), + block_parenthash character varying(66), + block_size bigint, + uncle_hash character varying(66), + node_id integer NOT NULL, + is_final boolean, + block_miner character varying(42), + block_extra_data character varying, + block_reward numeric, + block_uncles_reward numeric +); + + +-- +-- Name: blocks_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE blocks_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: blocks_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE blocks_id_seq OWNED BY blocks.id; + + +-- +-- Name: logs; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE logs ( + id integer NOT NULL, + block_number bigint, + address character varying(66), + tx_hash character varying(66), + index bigint, + topic0 character varying(66), + topic1 character varying(66), + topic2 character varying(66), + topic3 character varying(66), + data text, + receipt_id integer +); + + +-- +-- Name: logs_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE logs_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE logs_id_seq OWNED BY logs.id; + + +-- +-- Name: nodes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE nodes ( + id integer NOT NULL, + genesis_block character varying(66), + network_id numeric, + node_id character varying(128), + client_name character varying +); + + +-- +-- Name: nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE nodes_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: nodes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE nodes_id_seq OWNED BY nodes.id; + + +-- +-- Name: receipts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE receipts ( + id integer NOT NULL, + transaction_id integer NOT NULL, + contract_address character varying(42), + cumulative_gas_used numeric, + gas_used numeric, + state_root character varying(66), + status integer, + tx_hash character varying(66) +); + + +-- +-- Name: receipts_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE receipts_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: receipts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE receipts_id_seq OWNED BY receipts.id; + + +-- +-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE schema_migrations ( + version bigint NOT NULL, + dirty boolean NOT NULL +); + + +-- +-- Name: transactions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE transactions ( + id integer NOT NULL, + tx_hash character varying(66), + tx_nonce numeric, + tx_to character varying(66), + tx_gaslimit numeric, + tx_gasprice numeric, + tx_value numeric, + block_id integer NOT NULL, + tx_from character varying(66), + tx_input_data character varying +); + + +-- +-- Name: transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE transactions_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE transactions_id_seq OWNED BY transactions.id; + + +-- +-- Name: watched_contracts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE watched_contracts ( + contract_id integer NOT NULL, + contract_hash character varying(66), + contract_abi json +); + + +-- +-- Name: watched_contracts_contract_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE watched_contracts_contract_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: watched_contracts_contract_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE watched_contracts_contract_id_seq OWNED BY watched_contracts.contract_id; + + +-- +-- Name: blocks id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY blocks ALTER COLUMN id SET DEFAULT nextval('blocks_id_seq'::regclass); + + +-- +-- Name: logs id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY logs ALTER COLUMN id SET DEFAULT nextval('logs_id_seq'::regclass); + + +-- +-- Name: nodes id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY nodes ALTER COLUMN id SET DEFAULT nextval('nodes_id_seq'::regclass); + + +-- +-- Name: receipts id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY receipts ALTER COLUMN id SET DEFAULT nextval('receipts_id_seq'::regclass); + + +-- +-- Name: transactions id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY transactions ALTER COLUMN id SET DEFAULT nextval('transactions_id_seq'::regclass); + + +-- +-- Name: watched_contracts contract_id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY watched_contracts ALTER COLUMN contract_id SET DEFAULT nextval('watched_contracts_contract_id_seq'::regclass); + + +-- +-- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY blocks + ADD CONSTRAINT blocks_pkey PRIMARY KEY (id); + + +-- +-- Name: watched_contracts contract_hash_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY watched_contracts + ADD CONSTRAINT contract_hash_uc UNIQUE (contract_hash); + + +-- +-- Name: logs log_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY logs + ADD CONSTRAINT log_uc UNIQUE (block_number, index); + + +-- +-- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY logs + ADD CONSTRAINT logs_pkey PRIMARY KEY (id); + + +-- +-- Name: blocks node_id_block_number_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY blocks + ADD CONSTRAINT node_id_block_number_uc UNIQUE (block_number, node_id); + + +-- +-- Name: nodes node_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY nodes + ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id); + + +-- +-- Name: nodes nodes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY nodes + ADD CONSTRAINT nodes_pkey PRIMARY KEY (id); + + +-- +-- Name: receipts receipts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY receipts + ADD CONSTRAINT receipts_pkey PRIMARY KEY (id); + + +-- +-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY schema_migrations + ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); + + +-- +-- Name: transactions transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY transactions + ADD CONSTRAINT transactions_pkey PRIMARY KEY (id); + + +-- +-- Name: watched_contracts watched_contracts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY watched_contracts + ADD CONSTRAINT watched_contracts_pkey PRIMARY KEY (contract_id); + + +-- +-- Name: block_id_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX block_id_index ON transactions USING btree (block_id); + + +-- +-- Name: block_number_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX block_number_index ON blocks USING btree (block_number); + + +-- +-- Name: node_id_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX node_id_index ON blocks USING btree (node_id); + + +-- +-- Name: transaction_id_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX transaction_id_index ON receipts USING btree (transaction_id); + + +-- +-- Name: tx_from_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX tx_from_index ON transactions USING btree (tx_from); + + +-- +-- Name: tx_to_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX tx_to_index ON transactions USING btree (tx_to); + + +-- +-- Name: transactions blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY transactions + ADD CONSTRAINT blocks_fk FOREIGN KEY (block_id) REFERENCES blocks(id) ON DELETE CASCADE; + + +-- +-- Name: blocks node_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY blocks + ADD CONSTRAINT node_fk FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE; + + +-- +-- Name: logs receipts_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY logs + ADD CONSTRAINT receipts_fk FOREIGN KEY (receipt_id) REFERENCES receipts(id) ON DELETE CASCADE; + + +-- +-- Name: receipts transaction_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY receipts + ADD CONSTRAINT transaction_fk FOREIGN KEY (transaction_id) REFERENCES transactions(id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + From 6583ce72b82e3bea3d692a8855d102453ef8eddf Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Tue, 16 Jan 2018 09:28:19 -0600 Subject: [PATCH 6/7] Remove log_uc * Logs now are attached to receipt, so removing block + index unique constraint --- .../1516050071_add_log_fk_constraint.down.sql | 3 ++ .../1516050071_add_log_fk_constraint.up.sql | 2 ++ db/schema.sql | 8 ------ pkg/repositories/postgres.go | 28 ++----------------- pkg/repositories/testing/helpers.go | 24 ---------------- 5 files changed, 8 insertions(+), 57 deletions(-) diff --git a/db/migrations/1516050071_add_log_fk_constraint.down.sql b/db/migrations/1516050071_add_log_fk_constraint.down.sql index 6ce8446f..55779b51 100644 --- a/db/migrations/1516050071_add_log_fk_constraint.down.sql +++ b/db/migrations/1516050071_add_log_fk_constraint.down.sql @@ -6,4 +6,7 @@ ALTER TABLE logs ALTER TABLE logs DROP COLUMN receipt_id; +ALTER TABLE logs + ADD CONSTRAINT log_uc UNIQUE (block_number, index); + COMMIT; \ No newline at end of file diff --git a/db/migrations/1516050071_add_log_fk_constraint.up.sql b/db/migrations/1516050071_add_log_fk_constraint.up.sql index a0150d1a..60733b71 100644 --- a/db/migrations/1516050071_add_log_fk_constraint.up.sql +++ b/db/migrations/1516050071_add_log_fk_constraint.up.sql @@ -1,4 +1,6 @@ BEGIN; +ALTER TABLE logs + DROP CONSTRAINT log_uc; ALTER TABLE logs ADD COLUMN receipt_id INT; diff --git a/db/schema.sql b/db/schema.sql index 0b7f6190..e2e1ece3 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -324,14 +324,6 @@ ALTER TABLE ONLY watched_contracts ADD CONSTRAINT contract_hash_uc UNIQUE (contract_hash); --- --- Name: logs log_uc; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY logs - ADD CONSTRAINT log_uc UNIQUE (block_number, index); - - -- -- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- diff --git a/pkg/repositories/postgres.go b/pkg/repositories/postgres.go index ef63d396..14159547 100644 --- a/pkg/repositories/postgres.go +++ b/pkg/repositories/postgres.go @@ -351,21 +351,10 @@ func (repository Postgres) createReceipt(tx *sql.Tx, transactionId int, receipt func (repository Postgres) createLogs(tx *sql.Tx, logs []core.Log, receiptId int) error { for _, tlog := range logs { _, err := tx.Exec( - `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ON CONFLICT (index, block_number) - DO UPDATE - SET block_number = $1, - address = $2, - tx_hash = $3, - index = $4, - topic0 = $5, - topic1 = $6, - topic2 = $7, - topic3 = $8, - data = $9 + `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) `, - tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, + tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, receiptId, ) if err != nil { return ErrDBInsertFailed @@ -380,17 +369,6 @@ func (repository Postgres) CreateLogs(logs []core.Log) error { _, err := tx.Exec( `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ON CONFLICT (index, block_number) - DO UPDATE - SET block_number = $1, - address = $2, - tx_hash = $3, - index = $4, - topic0 = $5, - topic1 = $6, - topic2 = $7, - topic3 = $8, - data = $9 `, tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, ) diff --git a/pkg/repositories/testing/helpers.go b/pkg/repositories/testing/helpers.go index 6a89fd29..899b0987 100644 --- a/pkg/repositories/testing/helpers.go +++ b/pkg/repositories/testing/helpers.go @@ -422,30 +422,6 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Expect(log).To(BeNil()) }) - It("updates the log when log with when log with same block number and index is already present", func() { - repository.CreateLogs([]core.Log{{ - BlockNumber: 1, - Index: 0, - Address: "x123", - TxHash: "x456", - Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, - Data: "xABC", - }, - }) - repository.CreateLogs([]core.Log{{ - BlockNumber: 1, - Index: 0, - Address: "x123", - TxHash: "x456", - Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, - Data: "xXYZ", - }, - }) - - log := repository.FindLogs("x123", 1) - Expect(log[0].Data).To(Equal("xXYZ")) - }) - It("filters to the correct block number and address", func() { repository.CreateLogs([]core.Log{{ BlockNumber: 1, From 0ae25aff95cd46a58a3e1b17c1707e2278f4a5c7 Mon Sep 17 00:00:00 2001 From: Matt Krump Date: Tue, 16 Jan 2018 09:48:42 -0600 Subject: [PATCH 7/7] Remove getLogs entrypoint --- Gododir/main.go | 14 --------- README.md | 6 ---- cmd/get_logs/main.go | 74 -------------------------------------------- 3 files changed, 94 deletions(-) delete mode 100644 cmd/get_logs/main.go diff --git a/Gododir/main.go b/Gododir/main.go index 454e46e0..6243f67f 100644 --- a/Gododir/main.go +++ b/Gododir/main.go @@ -42,20 +42,6 @@ func tasks(p *do.Project) { do.M{"environment": environment, "startingNumber": startingNumber, "$in": "cmd/populate_blocks"}) }) - p.Task("getLogs", nil, func(context *do.Context) { - environment := parseEnvironment(context) - contractHash := context.Args.MayString("", "contract-hash", "c") - if contractHash == "" { - log.Fatalln("--contract-hash required") - } - context.Start(`go run main.go --environment={{.environment}} --contract-hash={{.contractHash}}`, - do.M{ - "environment": environment, - "contractHash": contractHash, - "$in": "cmd/get_logs", - }) - }) - p.Task("watchContract", nil, func(context *do.Context) { environment := parseEnvironment(context) contractHash := context.Args.MayString("", "contract-hash", "c") diff --git a/README.md b/README.md index 832c00ab..e05adda6 100644 --- a/README.md +++ b/README.md @@ -91,12 +91,6 @@ The name of the JSON file should correspond the contract's address. 2. Start watching the contract `godo watchContract -- --environment= --contract-hash=` 3. Request summary data `godo showContractSummary -- --environment= --contract-hash=` - -## Retrieving Contract Logs - -1. Get the logs for a specific contract - - `godo getLogs -- --environment= --contract-hash=` - ### Configuring Additional Environments You can create configuration files for additional environments. diff --git a/cmd/get_logs/main.go b/cmd/get_logs/main.go deleted file mode 100644 index 3e013882..00000000 --- a/cmd/get_logs/main.go +++ /dev/null @@ -1,74 +0,0 @@ -package main - -import ( - "log" - - "flag" - - "math/big" - - "time" - - "strings" - - "github.com/8thlight/vulcanizedb/cmd" - "github.com/8thlight/vulcanizedb/pkg/core" - "github.com/8thlight/vulcanizedb/pkg/geth" -) - -func min(a, b int64) int64 { - if a < b { - return a - } - return b -} - -const ( - windowSize = 24 - pollingInterval = 10 * time.Second -) - -func main() { - environment := flag.String("environment", "", "Environment name") - contractHash := flag.String("contract-hash", "", "Contract hash to show summary") - flag.Parse() - - ticker := time.NewTicker(pollingInterval) - defer ticker.Stop() - - contractHashLowered := strings.ToLower(*contractHash) - config := cmd.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) - - lastBlockNumber := blockchain.LastBlock().Int64() - stepSize := int64(1000) - - go func() { - for i := int64(0); i < lastBlockNumber; i = min(i+stepSize, lastBlockNumber) { - logs, err := blockchain.GetLogs(core.Contract{Hash: contractHashLowered}, big.NewInt(i), big.NewInt(i+stepSize)) - log.Println("Backfilling Logs:", i) - if err != nil { - log.Println(err) - } - repository.CreateLogs(logs) - } - }() - - done := make(chan struct{}) - go func() { done <- struct{}{} }() - for range ticker.C { - select { - case <-done: - go func() { - z := &big.Int{} - z.Sub(blockchain.LastBlock(), big.NewInt(25)) - log.Printf("Logs Window: %d - %d", z.Int64(), blockchain.LastBlock().Int64()) - logs, _ := blockchain.GetLogs(core.Contract{Hash: contractHashLowered}, z, blockchain.LastBlock()) - repository.CreateLogs(logs) - done <- struct{}{} - }() - default: - } - } -}