diff --git a/Gododir/main.go b/Gododir/main.go index 09b49130..6243f67f 100644 --- a/Gododir/main.go +++ b/Gododir/main.go @@ -42,20 +42,6 @@ func tasks(p *do.Project) { do.M{"environment": environment, "startingNumber": startingNumber, "$in": "cmd/populate_blocks"}) }) - p.Task("getLogs", nil, func(context *do.Context) { - environment := parseEnvironment(context) - contractHash := context.Args.MayString("", "contract-hash", "c") - if contractHash == "" { - log.Fatalln("--contract-hash required") - } - context.Start(`go run main.go --environment={{.environment}} --contract-hash={{.contractHash}}`, - do.M{ - "environment": environment, - "contractHash": contractHash, - "$in": "cmd/get_logs", - }) - }) - p.Task("watchContract", nil, func(context *do.Context) { environment := parseEnvironment(context) contractHash := context.Args.MayString("", "contract-hash", "c") @@ -79,7 +65,9 @@ func tasks(p *do.Project) { cfg := cmd.LoadConfig(environment) connectString := config.DbConnectionString(cfg.Database) migrate := fmt.Sprintf("migrate -database '%s' -path ./db/migrations up", connectString) + dumpSchema := fmt.Sprintf("pg_dump -O -s %s > db/schema.sql", cfg.Database.Name) context.Bash(migrate) + context.Bash(dumpSchema) }) p.Task("rollback", nil, func(context *do.Context) { @@ -87,7 +75,9 @@ func tasks(p *do.Project) { cfg := cmd.LoadConfig(environment) connectString := config.DbConnectionString(cfg.Database) migrate := fmt.Sprintf("migrate -database '%s' -path ./db/migrations down 1", connectString) + dumpSchema := fmt.Sprintf("pg_dump -O -s %s > db/schema.sql", cfg.Database.Name) context.Bash(migrate) + context.Bash(dumpSchema) }) p.Task("showContractSummary", nil, func(context *do.Context) { diff --git a/README.md b/README.md index 832c00ab..e05adda6 100644 --- a/README.md +++ b/README.md @@ -91,12 +91,6 @@ The name of the JSON file should correspond the contract's address. 2. Start watching the contract `godo watchContract -- --environment= --contract-hash=` 3. Request summary data `godo showContractSummary -- --environment= --contract-hash=` - -## Retrieving Contract Logs - -1. Get the logs for a specific contract - - `godo getLogs -- --environment= --contract-hash=` - ### Configuring Additional Environments You can create configuration files for additional environments. diff --git a/cmd/get_logs/main.go b/cmd/get_logs/main.go deleted file mode 100644 index 3e013882..00000000 --- a/cmd/get_logs/main.go +++ /dev/null @@ -1,74 +0,0 @@ -package main - -import ( - "log" - - "flag" - - "math/big" - - "time" - - "strings" - - "github.com/8thlight/vulcanizedb/cmd" - "github.com/8thlight/vulcanizedb/pkg/core" - "github.com/8thlight/vulcanizedb/pkg/geth" -) - -func min(a, b int64) int64 { - if a < b { - return a - } - return b -} - -const ( - windowSize = 24 - pollingInterval = 10 * time.Second -) - -func main() { - environment := flag.String("environment", "", "Environment name") - contractHash := flag.String("contract-hash", "", "Contract hash to show summary") - flag.Parse() - - ticker := time.NewTicker(pollingInterval) - defer ticker.Stop() - - contractHashLowered := strings.ToLower(*contractHash) - config := cmd.LoadConfig(*environment) - blockchain := geth.NewBlockchain(config.Client.IPCPath) - repository := cmd.LoadPostgres(config.Database, blockchain.Node()) - - lastBlockNumber := blockchain.LastBlock().Int64() - stepSize := int64(1000) - - go func() { - for i := int64(0); i < lastBlockNumber; i = min(i+stepSize, lastBlockNumber) { - logs, err := blockchain.GetLogs(core.Contract{Hash: contractHashLowered}, big.NewInt(i), big.NewInt(i+stepSize)) - log.Println("Backfilling Logs:", i) - if err != nil { - log.Println(err) - } - repository.CreateLogs(logs) - } - }() - - done := make(chan struct{}) - go func() { done <- struct{}{} }() - for range ticker.C { - select { - case <-done: - go func() { - z := &big.Int{} - z.Sub(blockchain.LastBlock(), big.NewInt(25)) - log.Printf("Logs Window: %d - %d", z.Int64(), blockchain.LastBlock().Int64()) - logs, _ := blockchain.GetLogs(core.Contract{Hash: contractHashLowered}, z, blockchain.LastBlock()) - repository.CreateLogs(logs) - done <- struct{}{} - }() - default: - } - } -} diff --git a/db/migrations/1516050071_add_log_fk_constraint.down.sql b/db/migrations/1516050071_add_log_fk_constraint.down.sql new file mode 100644 index 00000000..55779b51 --- /dev/null +++ b/db/migrations/1516050071_add_log_fk_constraint.down.sql @@ -0,0 +1,12 @@ +BEGIN; + +ALTER TABLE logs + DROP CONSTRAINT receipts_fk; + +ALTER TABLE logs + DROP COLUMN receipt_id; + +ALTER TABLE logs + ADD CONSTRAINT log_uc UNIQUE (block_number, index); + +COMMIT; \ No newline at end of file diff --git a/db/migrations/1516050071_add_log_fk_constraint.up.sql b/db/migrations/1516050071_add_log_fk_constraint.up.sql new file mode 100644 index 00000000..60733b71 --- /dev/null +++ b/db/migrations/1516050071_add_log_fk_constraint.up.sql @@ -0,0 +1,14 @@ +BEGIN; +ALTER TABLE logs + DROP CONSTRAINT log_uc; + +ALTER TABLE logs + ADD COLUMN receipt_id INT; + +ALTER TABLE logs + ADD CONSTRAINT receipts_fk +FOREIGN KEY (receipt_id) +REFERENCES receipts (id) +ON DELETE CASCADE; + +COMMIT; \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql new file mode 100644 index 00000000..e2e1ece3 --- /dev/null +++ b/db/schema.sql @@ -0,0 +1,468 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 10.1 +-- Dumped by pg_dump version 10.1 + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; + + +-- +-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; + + +SET search_path = public, pg_catalog; + +SET default_tablespace = ''; + +SET default_with_oids = false; + +-- +-- Name: blocks; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE blocks ( + block_number bigint, + block_gaslimit double precision, + block_gasused double precision, + block_time double precision, + id integer NOT NULL, + block_difficulty bigint, + block_hash character varying(66), + block_nonce character varying(20), + block_parenthash character varying(66), + block_size bigint, + uncle_hash character varying(66), + node_id integer NOT NULL, + is_final boolean, + block_miner character varying(42), + block_extra_data character varying, + block_reward numeric, + block_uncles_reward numeric +); + + +-- +-- Name: blocks_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE blocks_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: blocks_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE blocks_id_seq OWNED BY blocks.id; + + +-- +-- Name: logs; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE logs ( + id integer NOT NULL, + block_number bigint, + address character varying(66), + tx_hash character varying(66), + index bigint, + topic0 character varying(66), + topic1 character varying(66), + topic2 character varying(66), + topic3 character varying(66), + data text, + receipt_id integer +); + + +-- +-- Name: logs_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE logs_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE logs_id_seq OWNED BY logs.id; + + +-- +-- Name: nodes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE nodes ( + id integer NOT NULL, + genesis_block character varying(66), + network_id numeric, + node_id character varying(128), + client_name character varying +); + + +-- +-- Name: nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE nodes_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: nodes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE nodes_id_seq OWNED BY nodes.id; + + +-- +-- Name: receipts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE receipts ( + id integer NOT NULL, + transaction_id integer NOT NULL, + contract_address character varying(42), + cumulative_gas_used numeric, + gas_used numeric, + state_root character varying(66), + status integer, + tx_hash character varying(66) +); + + +-- +-- Name: receipts_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE receipts_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: receipts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE receipts_id_seq OWNED BY receipts.id; + + +-- +-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE schema_migrations ( + version bigint NOT NULL, + dirty boolean NOT NULL +); + + +-- +-- Name: transactions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE transactions ( + id integer NOT NULL, + tx_hash character varying(66), + tx_nonce numeric, + tx_to character varying(66), + tx_gaslimit numeric, + tx_gasprice numeric, + tx_value numeric, + block_id integer NOT NULL, + tx_from character varying(66), + tx_input_data character varying +); + + +-- +-- Name: transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE transactions_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE transactions_id_seq OWNED BY transactions.id; + + +-- +-- Name: watched_contracts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE watched_contracts ( + contract_id integer NOT NULL, + contract_hash character varying(66), + contract_abi json +); + + +-- +-- Name: watched_contracts_contract_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE watched_contracts_contract_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: watched_contracts_contract_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE watched_contracts_contract_id_seq OWNED BY watched_contracts.contract_id; + + +-- +-- Name: blocks id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY blocks ALTER COLUMN id SET DEFAULT nextval('blocks_id_seq'::regclass); + + +-- +-- Name: logs id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY logs ALTER COLUMN id SET DEFAULT nextval('logs_id_seq'::regclass); + + +-- +-- Name: nodes id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY nodes ALTER COLUMN id SET DEFAULT nextval('nodes_id_seq'::regclass); + + +-- +-- Name: receipts id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY receipts ALTER COLUMN id SET DEFAULT nextval('receipts_id_seq'::regclass); + + +-- +-- Name: transactions id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY transactions ALTER COLUMN id SET DEFAULT nextval('transactions_id_seq'::regclass); + + +-- +-- Name: watched_contracts contract_id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY watched_contracts ALTER COLUMN contract_id SET DEFAULT nextval('watched_contracts_contract_id_seq'::regclass); + + +-- +-- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY blocks + ADD CONSTRAINT blocks_pkey PRIMARY KEY (id); + + +-- +-- Name: watched_contracts contract_hash_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY watched_contracts + ADD CONSTRAINT contract_hash_uc UNIQUE (contract_hash); + + +-- +-- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY logs + ADD CONSTRAINT logs_pkey PRIMARY KEY (id); + + +-- +-- Name: blocks node_id_block_number_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY blocks + ADD CONSTRAINT node_id_block_number_uc UNIQUE (block_number, node_id); + + +-- +-- Name: nodes node_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY nodes + ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id); + + +-- +-- Name: nodes nodes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY nodes + ADD CONSTRAINT nodes_pkey PRIMARY KEY (id); + + +-- +-- Name: receipts receipts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY receipts + ADD CONSTRAINT receipts_pkey PRIMARY KEY (id); + + +-- +-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY schema_migrations + ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); + + +-- +-- Name: transactions transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY transactions + ADD CONSTRAINT transactions_pkey PRIMARY KEY (id); + + +-- +-- Name: watched_contracts watched_contracts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY watched_contracts + ADD CONSTRAINT watched_contracts_pkey PRIMARY KEY (contract_id); + + +-- +-- Name: block_id_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX block_id_index ON transactions USING btree (block_id); + + +-- +-- Name: block_number_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX block_number_index ON blocks USING btree (block_number); + + +-- +-- Name: node_id_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX node_id_index ON blocks USING btree (node_id); + + +-- +-- Name: transaction_id_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX transaction_id_index ON receipts USING btree (transaction_id); + + +-- +-- Name: tx_from_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX tx_from_index ON transactions USING btree (tx_from); + + +-- +-- Name: tx_to_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX tx_to_index ON transactions USING btree (tx_to); + + +-- +-- Name: transactions blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY transactions + ADD CONSTRAINT blocks_fk FOREIGN KEY (block_id) REFERENCES blocks(id) ON DELETE CASCADE; + + +-- +-- Name: blocks node_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY blocks + ADD CONSTRAINT node_fk FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE; + + +-- +-- Name: logs receipts_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY logs + ADD CONSTRAINT receipts_fk FOREIGN KEY (receipt_id) REFERENCES receipts(id) ON DELETE CASCADE; + + +-- +-- Name: receipts transaction_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY receipts + ADD CONSTRAINT transaction_fk FOREIGN KEY (transaction_id) REFERENCES transactions(id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + diff --git a/integration_test/contract_test.go b/integration_test/contract_test.go index 6d0af83f..33784973 100644 --- a/integration_test/contract_test.go +++ b/integration_test/contract_test.go @@ -107,8 +107,8 @@ var _ = Describe("Reading contracts", func() { expectedLogZero := core.Log{ BlockNumber: 4703824, TxHash: "0xf896bfd1eb539d881a1a31102b78de9f25cd591bf1fe1924b86148c0b205fd5d", - Address: "0xd26114cd6EE289AccF82350c8d8487fedB8A0C07", - Topics: map[int]string{ + Address: "0xd26114cd6ee289accf82350c8d8487fedb8a0c07", + Topics: core.Topics{ 0: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", 1: "0x000000000000000000000000fbb1b73c4f0bda4f67dca266ce6ef42f520fbb98", 2: "0x000000000000000000000000d26114cd6ee289accf82350c8d8487fedb8a0c07", diff --git a/pkg/core/log.go b/pkg/core/log.go index 9688b46c..984e0432 100644 --- a/pkg/core/log.go +++ b/pkg/core/log.go @@ -4,7 +4,7 @@ type Log struct { BlockNumber int64 TxHash string Address string - Topics map[int]string - Index int64 - Data string + Topics + Index int64 + Data string } diff --git a/pkg/core/topics.go b/pkg/core/topics.go new file mode 100644 index 00000000..2f1acb9d --- /dev/null +++ b/pkg/core/topics.go @@ -0,0 +1,3 @@ +package core + +type Topics [4]string diff --git a/pkg/geth/blockchain.go b/pkg/geth/blockchain.go index c519706f..5ff500c2 100644 --- a/pkg/geth/blockchain.go +++ b/pkg/geth/blockchain.go @@ -5,6 +5,8 @@ import ( "strings" + "log" + "github.com/8thlight/vulcanizedb/pkg/core" "github.com/8thlight/vulcanizedb/pkg/geth/node" "github.com/ethereum/go-ethereum" @@ -25,7 +27,10 @@ type Blockchain struct { func NewBlockchain(ipcPath string) *Blockchain { blockchain := Blockchain{} - rpcClient, _ := rpc.Dial(ipcPath) + rpcClient, err := rpc.Dial(ipcPath) + if err != nil { + log.Fatal(err) + } client := ethclient.NewClient(rpcClient) blockchain.node = node.Info(rpcClient) if infura := isInfuraNode(ipcPath); infura { @@ -57,7 +62,7 @@ func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumbe if err != nil { return []core.Log{}, err } - logs := GethLogsToCoreLogs(gethLogs) + logs := ToCoreLogs(gethLogs) return logs, nil } diff --git a/pkg/geth/log_to_core_log.go b/pkg/geth/log_to_core_log.go index f40983f4..7b24175f 100644 --- a/pkg/geth/log_to_core_log.go +++ b/pkg/geth/log_to_core_log.go @@ -1,19 +1,36 @@ package geth import ( + "strings" + "github.com/8thlight/vulcanizedb/pkg/core" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" ) -func LogToCoreLog(gethLog types.Log) core.Log { - topics := gethLog.Topics - var hexTopics = make(map[int]string) +func ToCoreLogs(gethLogs []types.Log) []core.Log { + var logs []core.Log + for _, log := range gethLogs { + log := ToCoreLog(log) + logs = append(logs, log) + } + return logs +} + +func makeTopics(topics []common.Hash) core.Topics { + var hexTopics core.Topics for i, topic := range topics { hexTopics[i] = topic.Hex() } + return hexTopics +} + +func ToCoreLog(gethLog types.Log) core.Log { + topics := gethLog.Topics + hexTopics := makeTopics(topics) return core.Log{ - Address: gethLog.Address.Hex(), + Address: strings.ToLower(gethLog.Address.Hex()), BlockNumber: int64(gethLog.BlockNumber), Topics: hexTopics, @@ -22,12 +39,3 @@ func LogToCoreLog(gethLog types.Log) core.Log { Data: hexutil.Encode(gethLog.Data), } } - -func GethLogsToCoreLogs(gethLogs []types.Log) []core.Log { - var logs []core.Log - for _, log := range gethLogs { - log := LogToCoreLog(log) - logs = append(logs, log) - } - return logs -} diff --git a/pkg/geth/log_to_core_log_test.go b/pkg/geth/log_to_core_log_test.go index 44448ce3..000276ff 100644 --- a/pkg/geth/log_to_core_log_test.go +++ b/pkg/geth/log_to_core_log_test.go @@ -1,6 +1,8 @@ package geth_test import ( + "strings" + "github.com/8thlight/vulcanizedb/pkg/core" "github.com/8thlight/vulcanizedb/pkg/geth" "github.com/ethereum/go-ethereum/common" @@ -14,7 +16,7 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { It("converts geth log to internal log format", func() { gethLog := types.Log{ - Address: common.HexToAddress("0xecf8f87f810ecf450940c9f60066b4a7a501d6a7"), + Address: common.HexToAddress("0x448a5065aeBB8E423F0896E6c5D525C040f59af3"), BlockHash: common.HexToHash("0x656c34545f90a730a19008c0e7a7cd4fb3895064b48d6d69761bd5abad681056"), BlockNumber: 2019236, Data: hexutil.MustDecode("0x000000000000000000000000000000000000000000000001a055690d9db80000"), @@ -28,18 +30,18 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { } expected := core.Log{ - Address: gethLog.Address.Hex(), + Address: strings.ToLower(gethLog.Address.Hex()), BlockNumber: int64(gethLog.BlockNumber), Data: hexutil.Encode(gethLog.Data), TxHash: gethLog.TxHash.Hex(), Index: 2, - Topics: map[int]string{ - 0: common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef").Hex(), - 1: common.HexToHash("0x00000000000000000000000080b2c9d7cbbf30a1b0fc8983c647d754c6525615").Hex(), + Topics: core.Topics{ + gethLog.Topics[0].Hex(), + gethLog.Topics[1].Hex(), }, } - coreLog := geth.LogToCoreLog(gethLog) + coreLog := geth.ToCoreLog(gethLog) Expect(coreLog.Address).To(Equal(expected.Address)) Expect(coreLog.BlockNumber).To(Equal(expected.BlockNumber)) @@ -79,10 +81,10 @@ var _ = Describe("Conversion of GethLog to core.Log", func() { }, } - expectedOne := geth.LogToCoreLog(gethLogOne) - expectedTwo := geth.LogToCoreLog(gethLogTwo) + expectedOne := geth.ToCoreLog(gethLogOne) + expectedTwo := geth.ToCoreLog(gethLogTwo) - coreLogs := geth.GethLogsToCoreLogs([]types.Log{gethLogOne, gethLogTwo}) + coreLogs := geth.ToCoreLogs([]types.Log{gethLogOne, gethLogTwo}) Expect(len(coreLogs)).To(Equal(2)) Expect(coreLogs[0]).To(Equal(expectedOne)) diff --git a/pkg/geth/receipt_to_core_receipt.go b/pkg/geth/receipt_to_core_receipt.go index b9130f4b..7117a6b7 100644 --- a/pkg/geth/receipt_to_core_receipt.go +++ b/pkg/geth/receipt_to_core_receipt.go @@ -49,7 +49,7 @@ func setContractAddress(gethReceipt *types.Receipt) string { func dereferenceLogs(gethReceipt *types.Receipt) []core.Log { logs := []core.Log{} for _, log := range gethReceipt.Logs { - logs = append(logs, LogToCoreLog(*log)) + logs = append(logs, ToCoreLog(*log)) } return logs } diff --git a/pkg/repositories/in_memory.go b/pkg/repositories/in_memory.go index 58cf4fe1..0a609231 100644 --- a/pkg/repositories/in_memory.go +++ b/pkg/repositories/in_memory.go @@ -102,6 +102,7 @@ func (repository *InMemory) CreateOrUpdateBlock(block core.Block) error { repository.blocks[block.Number] = block for _, transaction := range block.Transactions { repository.receipts[transaction.Hash] = transaction.Receipt + repository.logs[transaction.TxHash] = transaction.Logs } return nil } diff --git a/pkg/repositories/postgres.go b/pkg/repositories/postgres.go index 4418073a..14159547 100644 --- a/pkg/repositories/postgres.go +++ b/pkg/repositories/postgres.go @@ -62,35 +62,6 @@ func (repository Postgres) SetBlocksStatus(chainHead int64) { cutoff) } -func (repository Postgres) CreateLogs(logs []core.Log) error { - tx, _ := repository.Db.BeginTx(context.Background(), nil) - for _, tlog := range logs { - _, err := tx.Exec( - `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ON CONFLICT (index, block_number) - DO UPDATE - SET block_number = $1, - address = $2, - tx_hash = $3, - index = $4, - topic0 = $5, - topic1 = $6, - topic2 = $7, - topic3 = $8, - data = $9 - `, - tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, - ) - if err != nil { - tx.Rollback() - return ErrDBInsertFailed - } - } - tx.Commit() - return nil -} - func (repository Postgres) FindLogs(address string, blockNumber int64) []core.Log { logRows, _ := repository.Db.Query( `SELECT block_number, @@ -339,25 +310,74 @@ func (repository Postgres) createTransaction(tx *sql.Tx, blockId int64, transact if err != nil { return err } - if transaction.Receipt.TxHash != "" { - err = repository.createReceipt(tx, transactionId, transaction.Receipt) + if hasReceipt(transaction) { + receiptId, err := repository.createReceipt(tx, transactionId, transaction.Receipt) if err != nil { return err } + if hasLogs(transaction) { + err = repository.createLogs(tx, transaction.Receipt.Logs, receiptId) + if err != nil { + return err + } + } } return nil } -func (repository Postgres) createReceipt(tx *sql.Tx, transactionId int, receipt core.Receipt) error { +func hasLogs(transaction core.Transaction) bool { + return len(transaction.Receipt.Logs) > 0 +} + +func hasReceipt(transaction core.Transaction) bool { + return transaction.Receipt.TxHash != "" +} + +func (repository Postgres) createReceipt(tx *sql.Tx, transactionId int, receipt core.Receipt) (int, error) { //Not currently persisting log bloom filters - _, err := tx.Exec( + var receiptId int + err := tx.QueryRow( `INSERT INTO receipts (contract_address, tx_hash, cumulative_gas_used, gas_used, state_root, status, transaction_id) - VALUES ($1, $2, $3, $4, $5, $6, $7)`, - receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, transactionId) + VALUES ($1, $2, $3, $4, $5, $6, $7) + RETURNING id`, + receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, transactionId).Scan(&receiptId) if err != nil { - return err + return receiptId, err } + return receiptId, nil +} + +func (repository Postgres) createLogs(tx *sql.Tx, logs []core.Log, receiptId int) error { + for _, tlog := range logs { + _, err := tx.Exec( + `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + `, + tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, receiptId, + ) + if err != nil { + return ErrDBInsertFailed + } + } + return nil +} + +func (repository Postgres) CreateLogs(logs []core.Log) error { + tx, _ := repository.Db.BeginTx(context.Background(), nil) + for _, tlog := range logs { + _, err := tx.Exec( + `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + `, + tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, + ) + if err != nil { + tx.Rollback() + return ErrDBInsertFailed + } + } + tx.Commit() return nil } @@ -442,7 +462,7 @@ func (repository Postgres) loadLogs(logsRows *sql.Rows) []core.Log { var txHash string var index int64 var data string - topics := make([]string, 4) + var topics core.Topics logsRows.Scan(&blockNumber, &address, &txHash, &index, &topics[0], &topics[1], &topics[2], &topics[3], &data) log := core.Log{ BlockNumber: blockNumber, @@ -451,7 +471,6 @@ func (repository Postgres) loadLogs(logsRows *sql.Rows) []core.Log { Index: index, Data: data, } - log.Topics = make(map[int]string) for i, topic := range topics { log.Topics[i] = topic } diff --git a/pkg/repositories/testing/helpers.go b/pkg/repositories/testing/helpers.go index f2b42138..899b0987 100644 --- a/pkg/repositories/testing/helpers.go +++ b/pkg/repositories/testing/helpers.go @@ -399,7 +399,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, Data: "xabc", }}, ) @@ -422,37 +422,13 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Expect(log).To(BeNil()) }) - It("updates the log when log with when log with same block number and index is already present", func() { - repository.CreateLogs([]core.Log{{ - BlockNumber: 1, - Index: 0, - Address: "x123", - TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, - Data: "xABC", - }, - }) - repository.CreateLogs([]core.Log{{ - BlockNumber: 1, - Index: 0, - Address: "x123", - TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, - Data: "xXYZ", - }, - }) - - log := repository.FindLogs("x123", 1) - Expect(log[0].Data).To(Equal("xXYZ")) - }) - It("filters to the correct block number and address", func() { repository.CreateLogs([]core.Log{{ BlockNumber: 1, Index: 0, Address: "x123", TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, Data: "xabc", }}, ) @@ -461,7 +437,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 1, Address: "x123", TxHash: "x789", - Topics: map[int]string{0: "x111", 1: "x222", 2: "x333"}, + Topics: core.Topics{0: "x111", 1: "x222", 2: "x333"}, Data: "xdef", }}, ) @@ -470,7 +446,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. Index: 0, Address: "x123", TxHash: "x456", - Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"}, + Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"}, Data: "xabc", }}, ) @@ -504,6 +480,85 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories. {blockNumber: 1, Index: 1}}, )) }) + + It("saves the logs attached to a receipt", func() { + logs := []core.Log{{ + Address: "0x8a4774fe82c63484afef97ca8d89a6ea5e21f973", + BlockNumber: 4745407, + Data: "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000645a68669900000000000000000000000000000000000000000000003397684ab5869b0000000000000000000000000000000000000000000000000000000000005a36053200000000000000000000000099041f808d598b782d5a3e498681c2452a31da08", + Index: 86, + Topics: core.Topics{ + 0: "0x5a68669900000000000000000000000000000000000000000000000000000000", + 1: "0x000000000000000000000000d0148dad63f73ce6f1b6c607e3413dcf1ff5f030", + 2: "0x00000000000000000000000000000000000000000000003397684ab5869b0000", + 3: "0x000000000000000000000000000000000000000000000000000000005a360532", + }, + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + }, { + Address: "0x99041f808d598b782d5a3e498681c2452a31da08", + BlockNumber: 4745407, + Data: "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000418178358", + Index: 87, + Topics: core.Topics{ + 0: "0x1817835800000000000000000000000000000000000000000000000000000000", + 1: "0x0000000000000000000000008a4774fe82c63484afef97ca8d89a6ea5e21f973", + 2: "0x0000000000000000000000000000000000000000000000000000000000000000", + 3: "0x0000000000000000000000000000000000000000000000000000000000000000", + }, + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + }, { + Address: "0x99041f808d598b782d5a3e498681c2452a31da08", + BlockNumber: 4745407, + Data: "0x00000000000000000000000000000000000000000000003338f64c8423af4000", + Index: 88, + Topics: core.Topics{ + 0: "0x296ba4ca62c6c21c95e828080cb8aec7481b71390585605300a8a76f9e95b527", + }, + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + }, + } + receipt := core.Receipt{ + ContractAddress: "", + CumulativeGasUsed: 7481414, + GasUsed: 60711, + Logs: logs, + Bloom: "0x00000800000000000000001000000000000000400000000080000000000000000000400000010000000000000000000000000000040000000000000002000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000800004000000000000001000000000000000000000000000002000000480000000000000002000000000000000020000000000000000000000000000000000000000080000000000180000c00000000000000002000002000000040000000000000000000000000000010000000000020000000000000000000002000000000000000000000000400800000000000000000", + Status: 1, + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + } + transaction := + core.Transaction{ + Hash: receipt.TxHash, + Receipt: receipt, + } + + block := core.Block{Transactions: []core.Transaction{transaction}} + err := repository.CreateOrUpdateBlock(block) + Expect(err).To(Not(HaveOccurred())) + retrievedLogs := repository.FindLogs("0x99041f808d598b782d5a3e498681c2452a31da08", 4745407) + + expected := logs[1:] + Expect(retrievedLogs).To(Equal(expected)) + }) + + It("still saves receipts without logs", func() { + receipt := core.Receipt{ + TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", + } + transaction := core.Transaction{ + Hash: receipt.TxHash, + Receipt: receipt, + } + + block := core.Block{ + Transactions: []core.Transaction{transaction}, + } + repository.CreateOrUpdateBlock(block) + + _, err := repository.FindReceipt(receipt.TxHash) + + Expect(err).To(Not(HaveOccurred())) + }) }) Describe("Saving receipts", func() {