diff --git a/README.md b/README.md index 65b41dd3..05d450eb 100644 --- a/README.md +++ b/README.md @@ -54,9 +54,10 @@ Vulcanize DB is a set of tools that make it easier for developers to write appli - See `environments/infura.toml` to configure commands to run against infura, if a local node is unavailable ## Start syncing with postgres -Syncs VulcanizeDB with the configured Ethereum node. -1. Start node (**if fast syncing wait for initial sync to finish**) -1. In a separate terminal start vulcanize_db +Syncs VulcanizeDB with the configured Ethereum node, populating blocks, transactions, receipts, and logs. +This command is useful when you want to maintain a broad cache of what's happening on the blockchain. +1. Start Ethereum node (**if fast syncing your Ethereum node, wait for initial sync to finish**) +1. In a separate terminal start VulcanizeDB: - `./vulcanizedb sync --config --starting-block-number ` ## Alternatively, sync from Geth's underlying LevelDB @@ -69,6 +70,13 @@ Sync VulcanizeDB from the LevelDB underlying a Geth node. - `--ending-block-number`/`-e`: block number to sync to - `--all`/`-a`: sync all missing blocks +## Alternatively, sync in "light" mode +Syncs VulcanizeDB with the configured Ethereum node, populating only block headers. +This command is useful when you want a minimal baseline from which to track targeted data on the blockchain (e.g. individual smart contract storage values). +1. Start Ethereum node +2. In a separate terminal start VulcanizeDB: + - `./vulcanizedb lightSync --config --starting-block-number ` + ## Start full environment in docker by single command ### Geth Rinkeby diff --git a/cmd/coldImport.go b/cmd/coldImport.go index afb86ffa..d7265cb5 100644 --- a/cmd/coldImport.go +++ b/cmd/coldImport.go @@ -1,4 +1,4 @@ -// Copyright © 2018 Rob Mulholand +// Copyright © 2018 Vulcanize // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -44,8 +44,8 @@ Geth must be synced over all of the desired blocks and must not be running in or func init() { rootCmd.AddCommand(coldImportCmd) - coldImportCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Number for first block to cold import.") - coldImportCmd.Flags().Int64VarP(&endingBlockNumber, "ending-block-number", "e", 5500000, "Number for last block to cold import.") + coldImportCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "BlockNumber for first block to cold import.") + coldImportCmd.Flags().Int64VarP(&endingBlockNumber, "ending-block-number", "e", 5500000, "BlockNumber for last block to cold import.") coldImportCmd.Flags().BoolVarP(&syncAll, "all", "a", false, "Option to sync all missing blocks.") } @@ -82,8 +82,8 @@ func coldImport() { // init cold importer deps blockRepository := repositories.NewBlockRepository(&pgDB) receiptRepository := repositories.ReceiptRepository{DB: &pgDB} - transactionconverter := cold_db.NewColdDbTransactionConverter() - blockConverter := vulcCommon.NewBlockConverter(transactionconverter) + transactionConverter := cold_db.NewColdDbTransactionConverter() + blockConverter := vulcCommon.NewBlockConverter(transactionConverter) // init and execute cold importer coldImporter := cold_import.NewColdImporter(ethDB, blockRepository, receiptRepository, blockConverter) diff --git a/cmd/erc20.go b/cmd/erc20.go index 6bc3f375..9a9cacde 100644 --- a/cmd/erc20.go +++ b/cmd/erc20.go @@ -15,13 +15,20 @@ package cmd import ( + "log" + "time" + + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" "github.com/spf13/cobra" + "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" "github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/geth" - "log" - "time" + "github.com/vulcanize/vulcanizedb/pkg/geth/client" + vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" + "github.com/vulcanize/vulcanizedb/pkg/geth/node" ) // erc20Cmd represents the erc20 command @@ -49,14 +56,23 @@ Expects an ethereum node to be running and requires a .toml config file: func watchERC20s() { ticker := time.NewTicker(5 * time.Second) defer ticker.Stop() - blockchain := geth.NewBlockchain(ipc) - db, err := postgres.NewDB(databaseConfig, blockchain.Node()) + rawRpcClient, err := rpc.Dial(ipc) + if err != nil { + log.Fatal(err) + } + rpcClient := client.NewRpcClient(rawRpcClient, ipc) + ethClient := ethclient.NewClient(rawRpcClient) + client := client.NewEthClient(ethClient) + node := node.MakeNode(rpcClient) + transactionConverter := vRpc.NewRpcTransactionConverter(ethClient) + blockChain := geth.NewBlockChain(client, node, transactionConverter) + db, err := postgres.NewDB(databaseConfig, blockChain.Node()) if err != nil { log.Fatal("Failed to initialize database.") } watcher := shared.Watcher{ DB: *db, - Blockchain: blockchain, + Blockchain: blockChain, } watcher.AddTransformers(every_block.TransformerInitializers()) diff --git a/cmd/lightSync.go b/cmd/lightSync.go new file mode 100644 index 00000000..242c1e53 --- /dev/null +++ b/cmd/lightSync.go @@ -0,0 +1,107 @@ +// Copyright © 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "log" + "os" + "time" + + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" + "github.com/spf13/cobra" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/geth" + "github.com/vulcanize/vulcanizedb/pkg/geth/client" + vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" + "github.com/vulcanize/vulcanizedb/pkg/geth/node" + "github.com/vulcanize/vulcanizedb/pkg/history" + "github.com/vulcanize/vulcanizedb/utils" +) + +// lightSyncCmd represents the lightSync command +var lightSyncCmd = &cobra.Command{ + Use: "lightSync", + Short: "Syncs VulcanizeDB with local ethereum node's block headers", + Long: `Syncs VulcanizeDB with local ethereum node. Populates +Postgres with block headers. + +./vulcanizedb lightSync --starting-block-number 0 --config public.toml + +Expects ethereum node to be running and requires a .toml config: + + [database] + name = "vulcanize_public" + hostname = "localhost" + port = 5432 + + [client] + ipcPath = "/Users/user/Library/Ethereum/geth.ipc" +`, + Run: func(cmd *cobra.Command, args []string) { + lightSync() + }, +} + +func init() { + rootCmd.AddCommand(lightSyncCmd) + lightSyncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from") +} + +func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, missingBlocksPopulated chan int, startingBlockNumber int64) { + missingBlocksPopulated <- history.PopulateMissingHeaders(blockchain, headerRepository, startingBlockNumber) +} + +func lightSync() { + ticker := time.NewTicker(pollingInterval) + defer ticker.Stop() + rawRpcClient, err := rpc.Dial(ipc) + if err != nil { + log.Fatal(err) + } + rpcClient := client.NewRpcClient(rawRpcClient, ipc) + ethClient := ethclient.NewClient(rawRpcClient) + client := client.NewEthClient(ethClient) + node := node.MakeNode(rpcClient) + transactionConverter := vRpc.NewRpcTransactionConverter(client) + blockChain := geth.NewBlockChain(client, node, transactionConverter) + + lastBlock := blockChain.LastBlock().Int64() + if lastBlock == 0 { + log.Fatal("geth initial: state sync not finished") + } + if startingBlockNumber > lastBlock { + log.Fatal("starting block number > current block number") + } + + db := utils.LoadPostgres(databaseConfig, blockChain.Node()) + headerRepository := repositories.NewHeaderRepository(&db) + validator := history.NewHeaderValidator(blockChain, headerRepository, validationWindow) + missingBlocksPopulated := make(chan int) + go backFillAllHeaders(blockChain, headerRepository, missingBlocksPopulated, startingBlockNumber) + + for { + select { + case <-ticker.C: + window := validator.ValidateHeaders() + window.Log(os.Stdout) + case <-missingBlocksPopulated: + go backFillAllHeaders(blockChain, headerRepository, missingBlocksPopulated, startingBlockNumber) + } + } +} diff --git a/cmd/root.go b/cmd/root.go index 45a633ca..c0336edc 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,3 +1,17 @@ +// Copyright © 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd import ( diff --git a/cmd/sync.go b/cmd/sync.go index 42a2e111..99d89973 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -1,17 +1,35 @@ +// Copyright © 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd import ( + "log" "os" - "time" - "log" - + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" "github.com/spf13/cobra" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/geth" + "github.com/vulcanize/vulcanizedb/pkg/geth/client" + vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" + "github.com/vulcanize/vulcanizedb/pkg/geth/node" "github.com/vulcanize/vulcanizedb/pkg/history" "github.com/vulcanize/vulcanizedb/utils" ) @@ -19,9 +37,11 @@ import ( // syncCmd represents the sync command var syncCmd = &cobra.Command{ Use: "sync", - Short: "Syncs vulcanizedb with local ethereum node", - Long: `Syncs vulcanizedb with local ethereum node. -vulcanizedb sync --starting-block-number 0 --config public.toml + Short: "Syncs VulcanizeDB with local ethereum node", + Long: `Syncs VulcanizeDB with local ethereum node. Populates +Postgres with blocks, transactions, receipts, and logs. + +./vulcanizedb sync --starting-block-number 0 --config public.toml Expects ethereum node to be running and requires a .toml config: @@ -39,7 +59,8 @@ Expects ethereum node to be running and requires a .toml config: } const ( - pollingInterval = 7 * time.Second + pollingInterval = 7 * time.Second + validationWindow = 15 ) func init() { @@ -48,16 +69,25 @@ func init() { syncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from") } -func backFillAllBlocks(blockchain core.Blockchain, blockRepository datastore.BlockRepository, missingBlocksPopulated chan int, startingBlockNumber int64) { +func backFillAllBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, missingBlocksPopulated chan int, startingBlockNumber int64) { missingBlocksPopulated <- history.PopulateMissingBlocks(blockchain, blockRepository, startingBlockNumber) } func sync() { ticker := time.NewTicker(pollingInterval) defer ticker.Stop() - blockchain := geth.NewBlockchain(ipc) + rawRpcClient, err := rpc.Dial(ipc) + if err != nil { + log.Fatal(err) + } + rpcClient := client.NewRpcClient(rawRpcClient, ipc) + ethClient := ethclient.NewClient(rawRpcClient) + client := client.NewEthClient(ethClient) + node := node.MakeNode(rpcClient) + transactionConverter := vRpc.NewRpcTransactionConverter(ethClient) + blockChain := geth.NewBlockChain(client, node, transactionConverter) - lastBlock := blockchain.LastBlock().Int64() + lastBlock := blockChain.LastBlock().Int64() if lastBlock == 0 { log.Fatal("geth initial: state sync not finished") } @@ -65,19 +95,19 @@ func sync() { log.Fatal("starting block number > current block number") } - db := utils.LoadPostgres(databaseConfig, blockchain.Node()) + db := utils.LoadPostgres(databaseConfig, blockChain.Node()) blockRepository := repositories.NewBlockRepository(&db) - validator := history.NewBlockValidator(blockchain, blockRepository, 15) + validator := history.NewBlockValidator(blockChain, blockRepository, validationWindow) missingBlocksPopulated := make(chan int) - go backFillAllBlocks(blockchain, blockRepository, missingBlocksPopulated, startingBlockNumber) + go backFillAllBlocks(blockChain, blockRepository, missingBlocksPopulated, startingBlockNumber) for { select { case <-ticker.C: window := validator.ValidateBlocks() - validator.Log(os.Stdout, window) + window.Log(os.Stdout) case <-missingBlocksPopulated: - go backFillAllBlocks(blockchain, blockRepository, missingBlocksPopulated, startingBlockNumber) + go backFillAllBlocks(blockChain, blockRepository, missingBlocksPopulated, startingBlockNumber) } } } diff --git a/db/migrations/1531758090_create_headers_table.down.sql b/db/migrations/1531758090_create_headers_table.down.sql new file mode 100644 index 00000000..d9b283f9 --- /dev/null +++ b/db/migrations/1531758090_create_headers_table.down.sql @@ -0,0 +1 @@ +DROP TABLE public.headers; \ No newline at end of file diff --git a/db/migrations/1531758090_create_headers_table.up.sql b/db/migrations/1531758090_create_headers_table.up.sql new file mode 100644 index 00000000..31876183 --- /dev/null +++ b/db/migrations/1531758090_create_headers_table.up.sql @@ -0,0 +1,11 @@ +CREATE TABLE public.headers ( + id SERIAL PRIMARY KEY, + hash VARCHAR(66), + block_number BIGINT, + raw bytea, + eth_node_id INTEGER, + eth_node_fingerprint VARCHAR(128), + CONSTRAINT eth_nodes_fk FOREIGN KEY (eth_node_id) + REFERENCES eth_nodes (id) + ON DELETE CASCADE +); \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index c0599805..b9e03a52 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -121,6 +121,40 @@ CREATE TABLE public.eth_nodes ( ); +-- +-- Name: headers; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.headers ( + id integer NOT NULL, + hash character varying(66), + block_number bigint, + raw bytea, + eth_node_id integer, + eth_node_fingerprint character varying(128) +); + + +-- +-- Name: headers_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.headers_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: headers_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.headers_id_seq OWNED BY public.headers.id; + + -- -- Name: log_filters; Type: TABLE; Schema: public; Owner: - -- @@ -385,6 +419,13 @@ ALTER TABLE ONLY public.blocks ALTER COLUMN id SET DEFAULT nextval('public.block ALTER TABLE ONLY public.eth_nodes ALTER COLUMN id SET DEFAULT nextval('public.nodes_id_seq'::regclass); +-- +-- Name: headers id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.headers ALTER COLUMN id SET DEFAULT nextval('public.headers_id_seq'::regclass); + + -- -- Name: log_filters id; Type: DEFAULT; Schema: public; Owner: - -- @@ -459,6 +500,14 @@ ALTER TABLE ONLY public.eth_nodes ADD CONSTRAINT eth_node_uc UNIQUE (genesis_block, network_id, eth_node_id); +-- +-- Name: headers headers_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.headers + ADD CONSTRAINT headers_pkey PRIMARY KEY (id); + + -- -- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -574,6 +623,14 @@ ALTER TABLE ONLY public.token_supply ADD CONSTRAINT blocks_fk FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE; +-- +-- Name: headers eth_nodes_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.headers + ADD CONSTRAINT eth_nodes_fk FOREIGN KEY (eth_node_id) REFERENCES public.eth_nodes(id) ON DELETE CASCADE; + + -- -- Name: blocks node_fk; Type: FK CONSTRAINT; Schema: public; Owner: - -- diff --git a/examples/erc20_watcher/every_block/fetcher.go b/examples/erc20_watcher/every_block/fetcher.go index c42f1050..e644b0c5 100644 --- a/examples/erc20_watcher/every_block/fetcher.go +++ b/examples/erc20_watcher/every_block/fetcher.go @@ -16,24 +16,25 @@ package every_block import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "log" "math/big" + + "github.com/vulcanize/vulcanizedb/pkg/core" ) type ERC20FetcherInterface interface { FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error) - GetBlockchain() core.Blockchain + GetBlockChain() core.BlockChain } -func NewFetcher(blockchain core.Blockchain) Fetcher { +func NewFetcher(blockchain core.BlockChain) Fetcher { return Fetcher{ Blockchain: blockchain, } } type Fetcher struct { - Blockchain core.Blockchain + Blockchain core.BlockChain ContractAbi string ContractAddress string } @@ -65,6 +66,6 @@ func (f Fetcher) FetchSupplyOf(contractAbi string, contractAddress string, block return *result, nil } -func (f Fetcher) GetBlockchain() core.Blockchain { +func (f Fetcher) GetBlockChain() core.BlockChain { return f.Blockchain } diff --git a/examples/erc20_watcher/every_block/fetcher_test.go b/examples/erc20_watcher/every_block/fetcher_test.go index 13813ff6..9ca34486 100644 --- a/examples/erc20_watcher/every_block/fetcher_test.go +++ b/examples/erc20_watcher/every_block/fetcher_test.go @@ -15,46 +15,50 @@ package every_block_test import ( + "math/big" + + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/examples/constants" "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" - "github.com/vulcanize/vulcanizedb/examples/mocks" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/geth" - "math/big" + "github.com/vulcanize/vulcanizedb/pkg/geth/client" + rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" + "github.com/vulcanize/vulcanizedb/pkg/geth/node" ) var _ = Describe("ERC20 Fetcher", func() { blockNumber := int64(5502914) - infuraIPC := "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL" - realBlockchain := geth.NewBlockchain(infuraIPC) - realFetcher := every_block.NewFetcher(realBlockchain) - - fakeBlockchain := &mocks.Blockchain{} - testFetcher := every_block.NewFetcher(fakeBlockchain) - testAbi := "testAbi" - testContractAddress := "testContractAddress" - - errorBlockchain := &mocks.FailureBlockchain{} - errorFetcher := every_block.NewFetcher(errorBlockchain) - Describe("FetchSupplyOf", func() { It("fetches data from the blockchain with the correct arguments", func() { + fakeBlockchain := fakes.NewMockBlockChain() + testFetcher := every_block.NewFetcher(fakeBlockchain) + testAbi := "testAbi" + testContractAddress := "testContractAddress" _, err := testFetcher.FetchSupplyOf(testAbi, testContractAddress, blockNumber) Expect(err).NotTo(HaveOccurred()) - Expect(fakeBlockchain.FetchedAbi).To(Equal(testAbi)) - Expect(fakeBlockchain.FetchedContractAddress).To(Equal(testContractAddress)) - Expect(fakeBlockchain.FetchedMethod).To(Equal("totalSupply")) - Expect(fakeBlockchain.FetchedMethodArg).To(BeNil()) expectedResult := big.Int{} expected := &expectedResult - Expect(fakeBlockchain.FetchedResult).To(Equal(&expected)) - Expect(fakeBlockchain.FetchedBlockNumber).To(Equal(blockNumber)) + fakeBlockchain.AssertFetchContractDataCalledWith(testAbi, testContractAddress, "totalSupply", nil, &expected, blockNumber) }) It("fetches a token's total supply at the given block height", func() { + infuraIPC := "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL" + rawRpcClient, err := rpc.Dial(infuraIPC) + Expect(err).NotTo(HaveOccurred()) + rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC) + ethClient := ethclient.NewClient(rawRpcClient) + blockChainClient := client.NewEthClient(ethClient) + node := node.MakeNode(rpcClient) + transactionConverter := rpc2.NewRpcTransactionConverter(ethClient) + blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter) + realFetcher := every_block.NewFetcher(blockChain) result, err := realFetcher.FetchSupplyOf(constants.DaiAbiString, constants.DaiContractAddress, blockNumber) Expect(err).NotTo(HaveOccurred()) @@ -64,12 +68,15 @@ var _ = Describe("ERC20 Fetcher", func() { }) It("returns an error if the call to the blockchain fails", func() { + blockChain := fakes.NewMockBlockChain() + blockChain.SetFetchContractDataErr(fakes.FakeError) + errorFetcher := every_block.NewFetcher(blockChain) result, err := errorFetcher.FetchSupplyOf("", "", 0) Expect(result.String()).To(Equal("0")) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("totalSupply")) - Expect(err.Error()).To(ContainSubstring(mocks.TestError.Error())) + Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error())) }) }) }) diff --git a/examples/erc20_watcher/every_block/integration_test.go b/examples/erc20_watcher/every_block/integration_test.go index 68afc441..52dfa435 100644 --- a/examples/erc20_watcher/every_block/integration_test.go +++ b/examples/erc20_watcher/every_block/integration_test.go @@ -20,34 +20,35 @@ import ( "github.com/vulcanize/vulcanizedb/examples/constants" "github.com/vulcanize/vulcanizedb/examples/erc20_watcher" "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" - "github.com/vulcanize/vulcanizedb/examples/mocks" "github.com/vulcanize/vulcanizedb/examples/test_helpers" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "math/big" "strconv" ) -func setLastBlockOnChain(blockchain *mocks.Blockchain, blockNumber int64) { +func setLastBlockOnChain(blockChain *fakes.MockBlockChain, blockNumber int64) { blockNumberString := strconv.FormatInt(blockNumber, 10) lastBlockOnChain := big.Int{} lastBlockOnChain.SetString(blockNumberString, 10) - blockchain.SetLastBlock(&lastBlockOnChain) + blockChain.SetLastBlock(&lastBlockOnChain) } var _ = Describe("Everyblock transformers", func() { var db *postgres.DB - var blockchain mocks.Blockchain + var blockChain *fakes.MockBlockChain var blockNumber int64 var blockId int64 var err error BeforeEach(func() { + blockChain = fakes.NewMockBlockChain() blockNumber = erc20_watcher.DaiConfig.FirstBlock lastBlockNumber := blockNumber + 1 db = test_helpers.CreateNewDatabase() - setLastBlockOnChain(&blockchain, lastBlockNumber) + setLastBlockOnChain(blockChain, lastBlockNumber) blockRepository := repositories.NewBlockRepository(db) @@ -58,8 +59,8 @@ var _ = Describe("Everyblock transformers", func() { }) It("creates a token_supply record for each block in the given range", func() { - initializer := every_block.TokenSupplyTransformerInitializer{erc20_watcher.DaiConfig} - transformer := initializer.NewTokenSupplyTransformer(db, &blockchain) + initializer := every_block.TokenSupplyTransformerInitializer{Config: erc20_watcher.DaiConfig} + transformer := initializer.NewTokenSupplyTransformer(db, blockChain) transformer.Execute() var tokenSupplyCount int diff --git a/examples/erc20_watcher/every_block/transformer.go b/examples/erc20_watcher/every_block/transformer.go index 208df4cd..6ca2718b 100644 --- a/examples/erc20_watcher/every_block/transformer.go +++ b/examples/erc20_watcher/every_block/transformer.go @@ -38,7 +38,7 @@ type TokenSupplyTransformerInitializer struct { Config erc20_watcher.ContractConfig } -func (i TokenSupplyTransformerInitializer) NewTokenSupplyTransformer(db *postgres.DB, blockchain core.Blockchain) shared.Transformer { +func (i TokenSupplyTransformerInitializer) NewTokenSupplyTransformer(db *postgres.DB, blockchain core.BlockChain) shared.Transformer { fetcher := NewFetcher(blockchain) repository := TokenSupplyRepository{DB: db} transformer := Transformer{ @@ -74,7 +74,7 @@ func newTransformerError(err error, blockNumber int64, msg string) error { func (t Transformer) Execute() error { var upperBoundBlock int64 - blockchain := t.Fetcher.GetBlockchain() + blockchain := t.Fetcher.GetBlockChain() lastBlock := blockchain.LastBlock().Int64() if t.Config.LastBlock == -1 { diff --git a/examples/erc20_watcher/every_block/transformer_test.go b/examples/erc20_watcher/every_block/transformer_test.go index 41eeb684..a89a5fcc 100644 --- a/examples/erc20_watcher/every_block/transformer_test.go +++ b/examples/erc20_watcher/every_block/transformer_test.go @@ -21,6 +21,7 @@ import ( "github.com/vulcanize/vulcanizedb/examples/erc20_watcher" "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" "github.com/vulcanize/vulcanizedb/examples/mocks" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "math/big" "math/rand" "strconv" @@ -40,7 +41,7 @@ var _ = Describe("Everyblock transformer", func() { var fetcher mocks.Fetcher var repository mocks.TotalSupplyRepository var transformer every_block.Transformer - var blockchain mocks.Blockchain + var blockChain *fakes.MockBlockChain var initialSupply = "27647235749155415536952630" var initialSupplyPlusOne = "27647235749155415536952631" var initialSupplyPlusTwo = "27647235749155415536952632" @@ -48,9 +49,9 @@ var _ = Describe("Everyblock transformer", func() { var defaultLastBlock = big.Int{} BeforeEach(func() { - blockchain = mocks.Blockchain{} - blockchain.SetLastBlock(&defaultLastBlock) - fetcher = mocks.Fetcher{Blockchain: &blockchain} + blockChain = fakes.NewMockBlockChain() + blockChain.SetLastBlock(&defaultLastBlock) + fetcher = mocks.Fetcher{BlockChain: blockChain} fetcher.SetSupply(initialSupply) repository = mocks.TotalSupplyRepository{} repository.SetMissingBlocks([]int64{config.FirstBlock}) @@ -132,7 +133,7 @@ var _ = Describe("Everyblock transformer", func() { mostRecentBlock := big.Int{} mostRecentBlock.SetString(numberToString, 10) - blockchain.SetLastBlock(&mostRecentBlock) + blockChain.SetLastBlock(&mostRecentBlock) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) @@ -149,13 +150,14 @@ var _ = Describe("Everyblock transformer", func() { } err := transformer.Execute() Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("TestError")) + Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error())) Expect(err.Error()).To(ContainSubstring("fetching missing blocks")) }) - It("returns an error if the call to the blockchain fails", func() { - failureBlockchain := mocks.FailureBlockchain{} + It("returns an error if the call to the blockChain fails", func() { + failureBlockchain := fakes.NewMockBlockChain() failureBlockchain.SetLastBlock(&defaultLastBlock) + failureBlockchain.SetFetchContractDataErr(fakes.FakeError) fetcher := every_block.NewFetcher(failureBlockchain) transformer = every_block.Transformer{ Fetcher: &fetcher, @@ -163,7 +165,7 @@ var _ = Describe("Everyblock transformer", func() { } err := transformer.Execute() Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("TestError")) + Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error())) Expect(err.Error()).To(ContainSubstring("supply")) }) @@ -178,7 +180,7 @@ var _ = Describe("Everyblock transformer", func() { } err := transformer.Execute() Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("TestError")) + Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error())) Expect(err.Error()).To(ContainSubstring("supply")) }) }) diff --git a/examples/erc20_watcher/every_block/transformers.go b/examples/erc20_watcher/every_block/transformers.go index 2c886375..887610b0 100644 --- a/examples/erc20_watcher/every_block/transformers.go +++ b/examples/erc20_watcher/every_block/transformers.go @@ -15,8 +15,8 @@ package every_block import ( - "github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/examples/erc20_watcher" + "github.com/vulcanize/vulcanizedb/libraries/shared" ) func TransformerInitializers() []shared.TransformerInitializer { diff --git a/examples/mocks/mocks.go b/examples/mocks/mocks.go index c7fba520..b2944170 100644 --- a/examples/mocks/mocks.go +++ b/examples/mocks/mocks.go @@ -15,19 +15,18 @@ package mocks import ( - "errors" + "math/big" + "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" "github.com/vulcanize/vulcanizedb/pkg/core" - "math/big" + "github.com/vulcanize/vulcanizedb/pkg/fakes" ) -var TestError = errors.New("TestError") - type Fetcher struct { ContractAddress string Abi string FetchedBlocks []int64 - Blockchain core.Blockchain + BlockChain core.BlockChain supply big.Int } @@ -35,8 +34,8 @@ func (f *Fetcher) SetSupply(supply string) { f.supply.SetString(supply, 10) } -func (f Fetcher) GetBlockchain() core.Blockchain { - return f.Blockchain +func (f Fetcher) GetBlockChain() core.BlockChain { + return f.BlockChain } func (f *Fetcher) FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error) { @@ -80,7 +79,7 @@ type FailureRepository struct { func (fr *FailureRepository) Create(supply every_block.TokenSupply) error { if fr.createFail { - return TestError + return fakes.FakeError } else { return nil } @@ -88,7 +87,7 @@ func (fr *FailureRepository) Create(supply every_block.TokenSupply) error { func (fr *FailureRepository) MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) { if fr.missingBlocksFail { - return []int64{}, TestError + return []int64{}, fakes.FakeError } else { return fr.missingBlocksNumbers, nil } @@ -105,71 +104,3 @@ func (fr *FailureRepository) SetMissingBlocksFail(fail bool) { func (fr *FailureRepository) SetMissingBlocks(missingBlocks []int64) { fr.missingBlocksNumbers = missingBlocks } - -type Blockchain struct { - FetchedAbi string - FetchedContractAddress string - FetchedMethod string - FetchedMethodArg interface{} - FetchedResult interface{} - FetchedBlockNumber int64 - lastBlock *big.Int -} - -func (fb *Blockchain) FetchContractData(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) error { - fb.FetchedAbi = abiJSON - fb.FetchedContractAddress = address - fb.FetchedMethod = method - fb.FetchedMethodArg = methodArg - fb.FetchedResult = result - fb.FetchedBlockNumber = blockNumber - return nil -} - -func (fb *Blockchain) GetBlockByNumber(blockNumber int64) (core.Block, error) { - panic("implement me") -} - -func (fb *Blockchain) GetLogs(contract core.Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]core.Log, error) { - panic("implement me") -} - -func (fb *Blockchain) LastBlock() *big.Int { - return fb.lastBlock -} - -func (fb *Blockchain) Node() core.Node { - panic("implement me") -} - -func (fb *Blockchain) SetLastBlock(lastBlock *big.Int) { - fb.lastBlock = lastBlock -} - -type FailureBlockchain struct { - lastBlock *big.Int -} - -func (FailureBlockchain) FetchContractData(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) error { - return errors.New("TestError") -} - -func (FailureBlockchain) GetBlockByNumber(blockNumber int64) (core.Block, error) { - panic("implement me") -} - -func (FailureBlockchain) GetLogs(contract core.Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]core.Log, error) { - panic("implement me") -} - -func (fb FailureBlockchain) LastBlock() *big.Int { - return fb.lastBlock -} - -func (FailureBlockchain) Node() core.Node { - panic("implement me") -} - -func (fb *FailureBlockchain) SetLastBlock(lastBlock *big.Int) { - fb.lastBlock = lastBlock -} diff --git a/examples/test_helpers/database.go b/examples/test_helpers/database.go index 8c10f064..d4d5e9ad 100644 --- a/examples/test_helpers/database.go +++ b/examples/test_helpers/database.go @@ -35,21 +35,6 @@ type TransferDBRow struct { VulcanizeLogID int64 `db:"vulcanize_log_id"` } -func CreateLogRecord(db *postgres.DB, logRepository repositories.LogRepository, log core.Log) { - blockRepository := repositories.NewBlockRepository(db) - receiptRepository := repositories.ReceiptRepository{DB: db} - - blockNumber := log.BlockNumber - blockId, err := blockRepository.CreateOrUpdateBlock(core.Block{Number: blockNumber}) - Expect(err).NotTo(HaveOccurred()) - - receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{}) - Expect(err).NotTo(HaveOccurred()) - - err = logRepository.CreateLogs([]core.Log{log}, receiptId) - Expect(err).NotTo(HaveOccurred()) -} - func CreateNewDatabase() *postgres.DB { var node core.Node node = core.Node{ diff --git a/integration_test/block_rewards_test.go b/integration_test/block_rewards_test.go index 0fd7b65b..c4a09de0 100644 --- a/integration_test/block_rewards_test.go +++ b/integration_test/block_rewards_test.go @@ -1,24 +1,44 @@ package integration import ( + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/geth" + "github.com/vulcanize/vulcanizedb/pkg/geth/client" + vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" + "github.com/vulcanize/vulcanizedb/pkg/geth/node" "github.com/vulcanize/vulcanizedb/test_config" ) var _ = Describe("Rewards calculations", func() { It("calculates a block reward for a real block", func() { - blockchain := geth.NewBlockchain(test_config.InfuraClient.IPCPath) - block, err := blockchain.GetBlockByNumber(1071819) + rawRpcClient, err := rpc.Dial(test_config.InfuraClient.IPCPath) + Expect(err).NotTo(HaveOccurred()) + rpcClient := client.NewRpcClient(rawRpcClient, test_config.InfuraClient.IPCPath) + ethClient := ethclient.NewClient(rawRpcClient) + blockChainClient := client.NewEthClient(ethClient) + node := node.MakeNode(rpcClient) + transactionConverter := vRpc.NewRpcTransactionConverter(ethClient) + blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter) + block, err := blockChain.GetBlockByNumber(1071819) Expect(err).ToNot(HaveOccurred()) Expect(block.Reward).To(Equal(5.31355)) }) It("calculates an uncle reward for a real block", func() { - blockchain := geth.NewBlockchain(test_config.InfuraClient.IPCPath) - block, err := blockchain.GetBlockByNumber(1071819) + rawRpcClient, err := rpc.Dial(test_config.InfuraClient.IPCPath) + Expect(err).NotTo(HaveOccurred()) + rpcClient := client.NewRpcClient(rawRpcClient, test_config.InfuraClient.IPCPath) + ethClient := ethclient.NewClient(rawRpcClient) + blockChainClient := client.NewEthClient(ethClient) + node := node.MakeNode(rpcClient) + transactionConverter := vRpc.NewRpcTransactionConverter(ethClient) + blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter) + block, err := blockChain.GetBlockByNumber(1071819) Expect(err).ToNot(HaveOccurred()) Expect(block.UnclesReward).To(Equal(6.875)) }) diff --git a/integration_test/contract_test.go b/integration_test/contract_test.go index 2844c9f4..ad204edb 100644 --- a/integration_test/contract_test.go +++ b/integration_test/contract_test.go @@ -4,10 +4,15 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/geth" + "github.com/vulcanize/vulcanizedb/pkg/geth/client" + rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" + "github.com/vulcanize/vulcanizedb/pkg/geth/node" "github.com/vulcanize/vulcanizedb/pkg/geth/testing" "github.com/vulcanize/vulcanizedb/test_config" ) @@ -27,42 +32,60 @@ var _ = Describe("Reading contracts", func() { }, Index: 19, Data: "0x0000000000000000000000000000000000000000000000000c7d713b49da0000"} - blockchain := geth.NewBlockchain(test_config.InfuraClient.IPCPath) + rawRpcClient, err := rpc.Dial(test_config.InfuraClient.IPCPath) + Expect(err).NotTo(HaveOccurred()) + rpcClient := client.NewRpcClient(rawRpcClient, test_config.InfuraClient.IPCPath) + ethClient := ethclient.NewClient(rawRpcClient) + blockChainClient := client.NewEthClient(ethClient) + node := node.MakeNode(rpcClient) + transactionConverter := rpc2.NewRpcTransactionConverter(ethClient) + blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter) contract := testing.SampleContract() - logs, err := blockchain.GetLogs(contract, big.NewInt(4703824), nil) + logs, err := blockChain.GetLogs(contract, big.NewInt(4703824), nil) Expect(err).To(BeNil()) Expect(len(logs)).To(Equal(3)) Expect(logs[0]).To(Equal(expectedLogZero)) - }) It("returns and empty log array when no events for a given block / contract combo", func() { - blockchain := geth.NewBlockchain(test_config.InfuraClient.IPCPath) + rawRpcClient, err := rpc.Dial(test_config.InfuraClient.IPCPath) + Expect(err).NotTo(HaveOccurred()) + rpcClient := client.NewRpcClient(rawRpcClient, test_config.InfuraClient.IPCPath) + ethClient := ethclient.NewClient(rawRpcClient) + blockChainClient := client.NewEthClient(ethClient) + node := node.MakeNode(rpcClient) + transactionConverter := rpc2.NewRpcTransactionConverter(ethClient) + blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter) - logs, err := blockchain.GetLogs(core.Contract{Hash: "x123"}, big.NewInt(4703824), nil) + logs, err := blockChain.GetLogs(core.Contract{Hash: "x123"}, big.NewInt(4703824), nil) Expect(err).To(BeNil()) Expect(len(logs)).To(Equal(0)) }) - }) Describe("Fetching Contract data", func() { It("returns the correct attribute for a real contract", func() { - blockchain := geth.NewBlockchain(test_config.InfuraClient.IPCPath) + rawRpcClient, err := rpc.Dial(test_config.InfuraClient.IPCPath) + Expect(err).NotTo(HaveOccurred()) + rpcClient := client.NewRpcClient(rawRpcClient, test_config.InfuraClient.IPCPath) + ethClient := ethclient.NewClient(rawRpcClient) + blockChainClient := client.NewEthClient(ethClient) + node := node.MakeNode(rpcClient) + transactionConverter := rpc2.NewRpcTransactionConverter(ethClient) + blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter) contract := testing.SampleContract() var balance = new(big.Int) args := common.HexToHash("0xd26114cd6ee289accf82350c8d8487fedb8a0c07") - err := blockchain.FetchContractData(contract.Abi, "0xd26114cd6ee289accf82350c8d8487fedb8a0c07", "balanceOf", args, &balance, 5167471) + err = blockChain.FetchContractData(contract.Abi, "0xd26114cd6ee289accf82350c8d8487fedb8a0c07", "balanceOf", args, &balance, 5167471) Expect(err).NotTo(HaveOccurred()) expected := new(big.Int) expected.SetString("10897295492887612977137", 10) Expect(balance).To(Equal(expected)) }) }) - }) diff --git a/integration_test/geth_blockchain_test.go b/integration_test/geth_blockchain_test.go index 598837cb..9dc386eb 100644 --- a/integration_test/geth_blockchain_test.go +++ b/integration_test/geth_blockchain_test.go @@ -1,40 +1,49 @@ package integration_test import ( + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/inmemory" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/geth" + "github.com/vulcanize/vulcanizedb/pkg/geth/client" + rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" + "github.com/vulcanize/vulcanizedb/pkg/geth/node" "github.com/vulcanize/vulcanizedb/pkg/history" "github.com/vulcanize/vulcanizedb/test_config" ) var _ = Describe("Reading from the Geth blockchain", func() { - - var blockchain *geth.Blockchain - var inMemory *inmemory.InMemory + var blockChain *geth.BlockChain BeforeEach(func() { - blockchain = geth.NewBlockchain(test_config.InfuraClient.IPCPath) - inMemory = inmemory.NewInMemory() + rawRpcClient, err := rpc.Dial(test_config.InfuraClient.IPCPath) + Expect(err).NotTo(HaveOccurred()) + rpcClient := client.NewRpcClient(rawRpcClient, test_config.InfuraClient.IPCPath) + ethClient := ethclient.NewClient(rawRpcClient) + blockChainClient := client.NewEthClient(ethClient) + node := node.MakeNode(rpcClient) + transactionConverter := rpc2.NewRpcTransactionConverter(ethClient) + blockChain = geth.NewBlockChain(blockChainClient, node, transactionConverter) }) It("reads two blocks", func(done Done) { - blocks := &inmemory.BlockRepository{InMemory: inMemory} - lastBlock := blockchain.LastBlock() + blocks := fakes.NewMockBlockRepository() + lastBlock := blockChain.LastBlock() queriedBlocks := []int64{lastBlock.Int64() - 5, lastBlock.Int64() - 6} - history.RetrieveAndUpdateBlocks(blockchain, blocks, queriedBlocks) - Expect(blocks.BlockCount()).To(Equal(2)) + history.RetrieveAndUpdateBlocks(blockChain, blocks, queriedBlocks) + blocks.AssertCreateOrUpdateBlocksCallCountAndBlockNumbersEquals(2, []int64{lastBlock.Int64() - 5, lastBlock.Int64() - 6}) close(done) }, 30) It("retrieves the genesis block and first block", func(done Done) { - genesisBlock, err := blockchain.GetBlockByNumber(int64(0)) + genesisBlock, err := blockChain.GetBlockByNumber(int64(0)) Expect(err).ToNot(HaveOccurred()) - firstBlock, err := blockchain.GetBlockByNumber(int64(1)) + firstBlock, err := blockChain.GetBlockByNumber(int64(1)) Expect(err).ToNot(HaveOccurred()) - lastBlockNumber := blockchain.LastBlock() + lastBlockNumber := blockChain.LastBlock() Expect(genesisBlock.Number).To(Equal(int64(0))) Expect(firstBlock.Number).To(Equal(int64(1))) @@ -43,7 +52,7 @@ var _ = Describe("Reading from the Geth blockchain", func() { }, 15) It("retrieves the node info", func(done Done) { - node := blockchain.Node() + node := blockChain.Node() mainnetID := float64(1) Expect(node.GenesisBlock).ToNot(BeNil()) @@ -60,7 +69,7 @@ var _ = Describe("Reading from the Geth blockchain", func() { var blocks []core.Block n := 10 for i := 5327459; i > 5327459-n; i-- { - block, err := blockchain.GetBlockByNumber(int64(i)) + block, err := blockChain.GetBlockByNumber(int64(i)) Expect(err).ToNot(HaveOccurred()) blocks = append(blocks, block) } diff --git a/libraries/shared/transformer_interface.go b/libraries/shared/transformer_interface.go index 6a0c753a..a2fa6675 100644 --- a/libraries/shared/transformer_interface.go +++ b/libraries/shared/transformer_interface.go @@ -10,7 +10,7 @@ type Transformer interface { Execute() error } -type TransformerInitializer func(db *postgres.DB, blockchain core.Blockchain) Transformer +type TransformerInitializer func(db *postgres.DB, blockchain core.BlockChain) Transformer func HexToInt64(byteString string) int64 { value := common.HexToHash(byteString) diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index 5132bee3..1a5cd668 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -8,7 +8,7 @@ import ( type Watcher struct { Transformers []Transformer DB postgres.DB - Blockchain core.Blockchain + Blockchain core.BlockChain } func (watcher *Watcher) AddTransformers(us []TransformerInitializer) { diff --git a/libraries/shared/watcher_test.go b/libraries/shared/watcher_test.go index 729c4908..7add85d5 100644 --- a/libraries/shared/watcher_test.go +++ b/libraries/shared/watcher_test.go @@ -23,7 +23,7 @@ func (mh *MockTransformer) Execute() error { return nil } -func fakeTransformerInitializer(db *postgres.DB, blockchain core.Blockchain) shared.Transformer { +func fakeTransformerInitializer(db *postgres.DB, blockchain core.BlockChain) shared.Transformer { return &MockTransformer{} } diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index e67af221..fb56e96a 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -2,9 +2,10 @@ package core import "math/big" -type Blockchain interface { +type BlockChain interface { ContractDataFetcher GetBlockByNumber(blockNumber int64) (Block, error) + GetHeaderByNumber(blockNumber int64) (Header, error) GetLogs(contract Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]Log, error) LastBlock() *big.Int Node() Node diff --git a/pkg/core/eth_client.go b/pkg/core/eth_client.go new file mode 100644 index 00000000..186f222d --- /dev/null +++ b/pkg/core/eth_client.go @@ -0,0 +1,19 @@ +package core + +import ( + "context" + "math/big" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +type EthClient interface { + BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) + CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) + FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) + HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) + TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) + TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) +} diff --git a/pkg/core/header.go b/pkg/core/header.go new file mode 100644 index 00000000..21335b6f --- /dev/null +++ b/pkg/core/header.go @@ -0,0 +1,7 @@ +package core + +type Header struct { + BlockNumber int64 `db:"block_number"` + Hash string + Raw []byte +} diff --git a/pkg/core/rpc_client.go b/pkg/core/rpc_client.go new file mode 100644 index 00000000..d8b99354 --- /dev/null +++ b/pkg/core/rpc_client.go @@ -0,0 +1,9 @@ +package core + +import "context" + +type RpcClient interface { + CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error + IpcPath() string + SupportedModules() (map[string]string, error) +} diff --git a/pkg/datastore/inmemory/block_repository.go b/pkg/datastore/inmemory/block_repository.go deleted file mode 100644 index ee07efd2..00000000 --- a/pkg/datastore/inmemory/block_repository.go +++ /dev/null @@ -1,51 +0,0 @@ -package inmemory - -import ( - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore" -) - -type BlockRepository struct { - *InMemory -} - -func (blockRepository *BlockRepository) CreateOrUpdateBlock(block core.Block) (int64, error) { - blockRepository.CreateOrUpdateBlockCallCount++ - blockRepository.blocks[block.Number] = block - for _, transaction := range block.Transactions { - blockRepository.receipts[transaction.Hash] = transaction.Receipt - blockRepository.logs[transaction.TxHash] = transaction.Logs - } - return 0, nil -} - -func (blockRepository *BlockRepository) GetBlock(blockNumber int64) (core.Block, error) { - if block, ok := blockRepository.blocks[blockNumber]; ok { - return block, nil - } - return core.Block{}, datastore.ErrBlockDoesNotExist(blockNumber) -} - -func (blockRepository *BlockRepository) MissingBlockNumbers(startingBlockNumber int64, endingBlockNumber int64, nodeId string) []int64 { - missingNumbers := []int64{} - for blockNumber := int64(startingBlockNumber); blockNumber <= endingBlockNumber; blockNumber++ { - if _, ok := blockRepository.blocks[blockNumber]; !ok { - missingNumbers = append(missingNumbers, blockNumber) - } - } - return missingNumbers -} - -func (blockRepository *BlockRepository) SetBlocksStatus(chainHead int64) { - for key, block := range blockRepository.blocks { - if key < (chainHead - blocksFromHeadBeforeFinal) { - tmp := block - tmp.IsFinal = true - blockRepository.blocks[key] = tmp - } - } -} - -func (blockRepository *BlockRepository) BlockCount() int { - return len(blockRepository.blocks) -} diff --git a/pkg/datastore/inmemory/contract_repository.go b/pkg/datastore/inmemory/contract_repository.go deleted file mode 100644 index 6f759fd7..00000000 --- a/pkg/datastore/inmemory/contract_repository.go +++ /dev/null @@ -1,35 +0,0 @@ -package inmemory - -import ( - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore" -) - -type ContractRepostiory struct { - *InMemory -} - -func (contractRepository *ContractRepostiory) ContractExists(contractHash string) bool { - _, present := contractRepository.contracts[contractHash] - return present -} - -func (contractRepository *ContractRepostiory) GetContract(contractHash string) (core.Contract, error) { - contract, ok := contractRepository.contracts[contractHash] - if !ok { - return core.Contract{}, datastore.ErrContractDoesNotExist(contractHash) - } - for _, block := range contractRepository.blocks { - for _, transaction := range block.Transactions { - if transaction.To == contractHash { - contract.Transactions = append(contract.Transactions, transaction) - } - } - } - return contract, nil -} - -func (contractRepository *ContractRepostiory) CreateContract(contract core.Contract) error { - contractRepository.contracts[contract.Hash] = contract - return nil -} diff --git a/pkg/datastore/inmemory/in_memory.go b/pkg/datastore/inmemory/in_memory.go deleted file mode 100644 index 36e272f5..00000000 --- a/pkg/datastore/inmemory/in_memory.go +++ /dev/null @@ -1,30 +0,0 @@ -package inmemory - -import ( - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/filters" -) - -const ( - blocksFromHeadBeforeFinal = 20 -) - -type InMemory struct { - blocks map[int64]core.Block - receipts map[string]core.Receipt - contracts map[string]core.Contract - logs map[string][]core.Log - logFilters map[string]filters.LogFilter - CreateOrUpdateBlockCallCount int -} - -func NewInMemory() *InMemory { - return &InMemory{ - CreateOrUpdateBlockCallCount: 0, - blocks: make(map[int64]core.Block), - receipts: make(map[string]core.Receipt), - contracts: make(map[string]core.Contract), - logs: make(map[string][]core.Log), - logFilters: make(map[string]filters.LogFilter), - } -} diff --git a/pkg/datastore/postgres/repositories/header_repository.go b/pkg/datastore/postgres/repositories/header_repository.go new file mode 100644 index 00000000..7aa4deac --- /dev/null +++ b/pkg/datastore/postgres/repositories/header_repository.go @@ -0,0 +1,80 @@ +package repositories + +import ( + "database/sql" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type HeaderRepository struct { + database *postgres.DB +} + +func NewHeaderRepository(database *postgres.DB) HeaderRepository { + return HeaderRepository{database: database} +} + +func (repository HeaderRepository) CreateOrUpdateHeader(header core.Header) (int64, error) { + hash, err := repository.getHeaderHash(header) + if err != nil { + if headerDoesNotExist(err) { + return repository.insertHeader(header) + } + return 0, err + } + if headerMustBeReplaced(hash, header) { + return repository.replaceHeader(header) + } + return 0, err +} + +func (repository HeaderRepository) GetHeader(blockNumber int64) (core.Header, error) { + var header core.Header + err := repository.database.Get(&header, `SELECT block_number, hash, raw FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`, + blockNumber, repository.database.Node.ID) + return header, err +} + +func (repository HeaderRepository) MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) []int64 { + numbers := make([]int64, 0) + repository.database.Select(&numbers, `SELECT all_block_numbers + FROM ( + SELECT generate_series($1::INT, $2::INT) AS all_block_numbers) series + WHERE all_block_numbers NOT IN ( + SELECT block_number FROM headers WHERE eth_node_fingerprint = $3 + ) `, + startingBlockNumber, endingBlockNumber, nodeID) + return numbers +} + +func headerMustBeReplaced(hash string, header core.Header) bool { + return hash != header.Hash +} + +func headerDoesNotExist(err error) bool { + return err == sql.ErrNoRows +} + +func (repository HeaderRepository) getHeaderHash(header core.Header) (string, error) { + var hash string + err := repository.database.Get(&hash, `SELECT hash FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`, + header.BlockNumber, repository.database.Node.ID) + return hash, err +} + +func (repository HeaderRepository) insertHeader(header core.Header) (int64, error) { + var headerId int64 + err := repository.database.QueryRowx( + `INSERT INTO public.headers (block_number, hash, raw, eth_node_id, eth_node_fingerprint) VALUES ($1, $2, $3, $4, $5) RETURNING id`, + header.BlockNumber, header.Hash, header.Raw, repository.database.NodeID, repository.database.Node.ID).Scan(&headerId) + return headerId, err +} + +func (repository HeaderRepository) replaceHeader(header core.Header) (int64, error) { + _, err := repository.database.Exec(`DELETE FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`, + header.BlockNumber, repository.database.Node.ID) + if err != nil { + return 0, err + } + return repository.insertHeader(header) +} diff --git a/pkg/datastore/postgres/repositories/header_repository_test.go b/pkg/datastore/postgres/repositories/header_repository_test.go new file mode 100644 index 00000000..49a74af3 --- /dev/null +++ b/pkg/datastore/postgres/repositories/header_repository_test.go @@ -0,0 +1,248 @@ +package repositories_test + +import ( + "database/sql" + "github.com/ethereum/go-ethereum/common" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("Block header repository", func() { + Describe("creating or updating a header", func() { + + It("adds a header", func() { + node := core.Node{} + db := test_config.NewTestDB(node) + repo := repositories.NewHeaderRepository(db) + header := core.Header{ + BlockNumber: 100, + Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(), + Raw: []byte{1, 2, 3, 4, 5}, + } + + _, err := repo.CreateOrUpdateHeader(header) + + Expect(err).NotTo(HaveOccurred()) + var dbHeader core.Header + err = db.Get(&dbHeader, `SELECT block_number, hash, raw FROM public.headers WHERE block_number = $1`, header.BlockNumber) + Expect(err).NotTo(HaveOccurred()) + Expect(dbHeader.BlockNumber).To(Equal(header.BlockNumber)) + Expect(dbHeader.Hash).To(Equal(header.Hash)) + Expect(dbHeader.Raw).To(Equal(header.Raw)) + }) + + It("adds node data to header", func() { + node := core.Node{ID: "EthNodeFingerprint"} + db := test_config.NewTestDB(node) + repo := repositories.NewHeaderRepository(db) + header := core.Header{BlockNumber: 100} + + _, err := repo.CreateOrUpdateHeader(header) + + Expect(err).NotTo(HaveOccurred()) + var ethNodeId int64 + err = db.Get(ðNodeId, `SELECT eth_node_id FROM public.headers WHERE block_number = $1`, header.BlockNumber) + Expect(err).NotTo(HaveOccurred()) + Expect(ethNodeId).To(Equal(db.NodeID)) + var ethNodeFingerprint string + err = db.Get(ðNodeFingerprint, `SELECT eth_node_fingerprint FROM public.headers WHERE block_number = $1`, header.BlockNumber) + Expect(err).NotTo(HaveOccurred()) + Expect(ethNodeFingerprint).To(Equal(db.Node.ID)) + }) + + It("does not duplicate headers", func() { + node := core.Node{} + db := test_config.NewTestDB(node) + repo := repositories.NewHeaderRepository(db) + header := core.Header{ + BlockNumber: 100, + Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(), + Raw: []byte{1, 2, 3, 4, 5}, + } + + _, err := repo.CreateOrUpdateHeader(header) + Expect(err).NotTo(HaveOccurred()) + + _, err = repo.CreateOrUpdateHeader(header) + Expect(err).NotTo(HaveOccurred()) + + var dbHeaders []core.Header + err = db.Select(&dbHeaders, `SELECT block_number, hash, raw FROM public.headers WHERE block_number = $1`, header.BlockNumber) + Expect(err).NotTo(HaveOccurred()) + Expect(len(dbHeaders)).To(Equal(1)) + }) + + It("replaces header if hash is different", func() { + node := core.Node{} + db := test_config.NewTestDB(node) + repo := repositories.NewHeaderRepository(db) + header := core.Header{ + BlockNumber: 100, + Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(), + Raw: []byte{1, 2, 3, 4, 5}, + } + _, err := repo.CreateOrUpdateHeader(header) + Expect(err).NotTo(HaveOccurred()) + headerTwo := core.Header{ + BlockNumber: header.BlockNumber, + Hash: common.BytesToHash([]byte{5, 4, 3, 2, 1}).Hex(), + Raw: []byte{5, 4, 3, 2, 1}, + } + + _, err = repo.CreateOrUpdateHeader(headerTwo) + + Expect(err).NotTo(HaveOccurred()) + var dbHeader core.Header + err = db.Get(&dbHeader, `SELECT block_number, hash, raw FROM headers WHERE block_number = $1`, header.BlockNumber) + Expect(err).NotTo(HaveOccurred()) + Expect(dbHeader.Hash).To(Equal(headerTwo.Hash)) + Expect(dbHeader.Raw).To(Equal(headerTwo.Raw)) + }) + + It("does not replace header if node fingerprint is different", func() { + node := core.Node{ID: "Fingerprint"} + db := test_config.NewTestDB(node) + repo := repositories.NewHeaderRepository(db) + header := core.Header{ + BlockNumber: 100, + Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(), + Raw: []byte{1, 2, 3, 4, 5}, + } + _, err := repo.CreateOrUpdateHeader(header) + nodeTwo := core.Node{ID: "FingerprintTwo"} + dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo) + Expect(err).NotTo(HaveOccurred()) + repoTwo := repositories.NewHeaderRepository(dbTwo) + headerTwo := core.Header{ + BlockNumber: header.BlockNumber, + Hash: common.BytesToHash([]byte{5, 4, 3, 2, 1}).Hex(), + Raw: []byte{5, 4, 3, 2, 1}, + } + + _, err = repoTwo.CreateOrUpdateHeader(headerTwo) + + Expect(err).NotTo(HaveOccurred()) + var dbHeaders []core.Header + err = dbTwo.Select(&dbHeaders, `SELECT block_number, hash, raw FROM headers WHERE block_number = $1`, header.BlockNumber) + Expect(err).NotTo(HaveOccurred()) + Expect(len(dbHeaders)).To(Equal(2)) + }) + + It("only replaces header with matching node fingerprint", func() { + node := core.Node{ID: "Fingerprint"} + db := test_config.NewTestDB(node) + repo := repositories.NewHeaderRepository(db) + header := core.Header{ + BlockNumber: 100, + Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(), + Raw: []byte{1, 2, 3, 4, 5}, + } + _, err := repo.CreateOrUpdateHeader(header) + nodeTwo := core.Node{ID: "FingerprintTwo"} + dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo) + Expect(err).NotTo(HaveOccurred()) + repoTwo := repositories.NewHeaderRepository(dbTwo) + headerTwo := core.Header{ + BlockNumber: header.BlockNumber, + Hash: common.BytesToHash([]byte{5, 4, 3, 2, 1}).Hex(), + Raw: []byte{5, 4, 3, 2, 1}, + } + _, err = repoTwo.CreateOrUpdateHeader(headerTwo) + headerThree := core.Header{ + BlockNumber: header.BlockNumber, + Hash: common.BytesToHash([]byte{1, 1, 1, 1, 1}).Hex(), + Raw: []byte{1, 1, 1, 1, 1}, + } + + _, err = repoTwo.CreateOrUpdateHeader(headerThree) + + Expect(err).NotTo(HaveOccurred()) + var dbHeaders []core.Header + err = dbTwo.Select(&dbHeaders, `SELECT block_number, hash, raw FROM headers WHERE block_number = $1`, header.BlockNumber) + Expect(err).NotTo(HaveOccurred()) + Expect(len(dbHeaders)).To(Equal(2)) + Expect(dbHeaders[0].Hash).To(Or(Equal(header.Hash), Equal(headerThree.Hash))) + Expect(dbHeaders[1].Hash).To(Or(Equal(header.Hash), Equal(headerThree.Hash))) + Expect(dbHeaders[0].Raw).To(Or(Equal(header.Raw), Equal(headerThree.Raw))) + Expect(dbHeaders[1].Raw).To(Or(Equal(header.Raw), Equal(headerThree.Raw))) + }) + }) + + Describe("Getting a header", func() { + It("returns header if it exists", func() { + node := core.Node{} + db := test_config.NewTestDB(node) + repo := repositories.NewHeaderRepository(db) + header := core.Header{ + BlockNumber: 100, + Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(), + Raw: []byte{1, 2, 3, 4, 5}, + } + _, err := repo.CreateOrUpdateHeader(header) + Expect(err).NotTo(HaveOccurred()) + + dbHeader, err := repo.GetHeader(header.BlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(dbHeader).To(Equal(header)) + }) + + It("does not return header for a different node fingerprint", func() { + node := core.Node{} + db := test_config.NewTestDB(node) + repo := repositories.NewHeaderRepository(db) + header := core.Header{ + BlockNumber: 100, + Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(), + Raw: []byte{1, 2, 3, 4, 5}, + } + _, err := repo.CreateOrUpdateHeader(header) + Expect(err).NotTo(HaveOccurred()) + nodeTwo := core.Node{ID: "NodeFingerprintTwo"} + dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo) + Expect(err).NotTo(HaveOccurred()) + repoTwo := repositories.NewHeaderRepository(dbTwo) + + _, err = repoTwo.GetHeader(header.BlockNumber) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(sql.ErrNoRows)) + }) + }) + + Describe("Getting missing headers", func() { + It("returns block numbers for headers not in the database", func() { + node := core.Node{} + db := test_config.NewTestDB(node) + repo := repositories.NewHeaderRepository(db) + repo.CreateOrUpdateHeader(core.Header{BlockNumber: 1}) + repo.CreateOrUpdateHeader(core.Header{BlockNumber: 3}) + repo.CreateOrUpdateHeader(core.Header{BlockNumber: 5}) + + missingBlockNumbers := repo.MissingBlockNumbers(1, 5, node.ID) + + Expect(missingBlockNumbers).To(ConsistOf([]int64{2, 4})) + }) + + It("does not count headers created by a different node fingerprint", func() { + node := core.Node{ID: "NodeFingerprint"} + db := test_config.NewTestDB(node) + repo := repositories.NewHeaderRepository(db) + repo.CreateOrUpdateHeader(core.Header{BlockNumber: 1}) + repo.CreateOrUpdateHeader(core.Header{BlockNumber: 3}) + repo.CreateOrUpdateHeader(core.Header{BlockNumber: 5}) + nodeTwo := core.Node{ID: "NodeFingerprintTwo"} + dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo) + Expect(err).NotTo(HaveOccurred()) + repoTwo := repositories.NewHeaderRepository(dbTwo) + + missingBlockNumbers := repoTwo.MissingBlockNumbers(1, 5, nodeTwo.ID) + + Expect(missingBlockNumbers).To(ConsistOf([]int64{1, 2, 3, 4, 5})) + }) + }) +}) diff --git a/pkg/datastore/repository.go b/pkg/datastore/repository.go index 4b9592e8..fd8b59cd 100644 --- a/pkg/datastore/repository.go +++ b/pkg/datastore/repository.go @@ -14,7 +14,7 @@ var ErrBlockDoesNotExist = func(blockNumber int64) error { type BlockRepository interface { CreateOrUpdateBlock(block core.Block) (int64, error) GetBlock(blockNumber int64) (core.Block, error) - MissingBlockNumbers(startingBlockNumber int64, endingBlockNumber int64, nodeId string) []int64 + MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) []int64 SetBlocksStatus(chainHead int64) } @@ -37,6 +37,12 @@ type FilterRepository interface { GetFilter(name string) (filters.LogFilter, error) } +type HeaderRepository interface { + CreateOrUpdateHeader(header core.Header) (int64, error) + GetHeader(blockNumber int64) (core.Header, error) + MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) []int64 +} + type LogRepository interface { CreateLogs(logs []core.Log, receiptId int64) error GetLogs(address string, blockNumber int64) []core.Log diff --git a/pkg/fakes/blockchain.go b/pkg/fakes/blockchain.go deleted file mode 100644 index ffc8d1ab..00000000 --- a/pkg/fakes/blockchain.go +++ /dev/null @@ -1,76 +0,0 @@ -package fakes - -import ( - "math/big" - - "github.com/vulcanize/vulcanizedb/pkg/core" -) - -type Blockchain struct { - logs map[string][]core.Log - blocks map[int64]core.Block - contractAttributes map[string]map[string]string - blocksChannel chan core.Block - WasToldToStop bool - node core.Node - ContractReturnValue []byte - err error -} - -func (blockchain *Blockchain) FetchContractData(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) error { - panic("implement me") -} - -func (blockchain *Blockchain) CallContract(contractHash string, input []byte, blockNumber *big.Int) ([]byte, error) { - return blockchain.ContractReturnValue, nil -} - -func (blockchain *Blockchain) LastBlock() *big.Int { - var max int64 - for blockNumber := range blockchain.blocks { - if blockNumber > max { - max = blockNumber - } - } - return big.NewInt(max) -} - -func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlock *big.Int, endingBlock *big.Int) ([]core.Log, error) { - return blockchain.logs[contract.Hash], nil -} - -func (blockchain *Blockchain) Node() core.Node { - return blockchain.node -} - -func NewBlockchain(err error) *Blockchain { - return &Blockchain{ - blocks: make(map[int64]core.Block), - logs: make(map[string][]core.Log), - contractAttributes: make(map[string]map[string]string), - node: core.Node{GenesisBlock: "GENESIS", NetworkID: 1, ID: "x123", ClientName: "Geth"}, - err: err, - } -} - -func NewBlockchainWithBlocks(blocks []core.Block) *Blockchain { - blockNumberToBlocks := make(map[int64]core.Block) - for _, block := range blocks { - blockNumberToBlocks[block.Number] = block - } - return &Blockchain{ - blocks: blockNumberToBlocks, - } -} - -func (blockchain *Blockchain) GetBlockByNumber(blockNumber int64) (core.Block, error) { - if blockchain.err != nil { - return core.Block{}, blockchain.err - } - return blockchain.blocks[blockNumber], nil -} - -func (blockchain *Blockchain) AddBlock(block core.Block) { - blockchain.blocks[block.Number] = block - blockchain.blocksChannel <- block -} diff --git a/pkg/fakes/data.go b/pkg/fakes/data.go new file mode 100644 index 00000000..22f1b601 --- /dev/null +++ b/pkg/fakes/data.go @@ -0,0 +1,5 @@ +package fakes + +import "errors" + +var FakeError = errors.New("failed") diff --git a/pkg/fakes/mock_block_repository.go b/pkg/fakes/mock_block_repository.go index 6391c198..7c05bfc0 100644 --- a/pkg/fakes/mock_block_repository.go +++ b/pkg/fakes/mock_block_repository.go @@ -7,14 +7,16 @@ import ( ) type MockBlockRepository struct { + createOrUpdateBlockCallCount int createOrUpdateBlockCalled bool createOrUpdateBlockPassedBlock core.Block - createOrUpdateBlockReturnInt int64 + createOrUpdateBlockPassedBlockNumbers []int64 createOrUpdateBlockReturnErr error + createOrUpdateBlockReturnInt int64 missingBlockNumbersCalled bool - missingBlockNumbersPassedStartingBlockNumber int64 missingBlockNumbersPassedEndingBlockNumber int64 missingBlockNumbersPassedNodeId string + missingBlockNumbersPassedStartingBlockNumber int64 missingBlockNumbersReturnArray []int64 setBlockStatusCalled bool setBlockStatusPassedChainHead int64 @@ -22,65 +24,78 @@ type MockBlockRepository struct { func NewMockBlockRepository() *MockBlockRepository { return &MockBlockRepository{ + createOrUpdateBlockCallCount: 0, createOrUpdateBlockCalled: false, createOrUpdateBlockPassedBlock: core.Block{}, - createOrUpdateBlockReturnInt: 0, + createOrUpdateBlockPassedBlockNumbers: nil, createOrUpdateBlockReturnErr: nil, + createOrUpdateBlockReturnInt: 0, missingBlockNumbersCalled: false, - missingBlockNumbersPassedStartingBlockNumber: 0, missingBlockNumbersPassedEndingBlockNumber: 0, missingBlockNumbersPassedNodeId: "", + missingBlockNumbersPassedStartingBlockNumber: 0, missingBlockNumbersReturnArray: nil, setBlockStatusCalled: false, setBlockStatusPassedChainHead: 0, } } -func (mbr *MockBlockRepository) SetCreateOrUpdateBlockReturnVals(i int64, err error) { - mbr.createOrUpdateBlockReturnInt = i - mbr.createOrUpdateBlockReturnErr = err +func (repository *MockBlockRepository) SetCreateOrUpdateBlockReturnVals(i int64, err error) { + repository.createOrUpdateBlockReturnInt = i + repository.createOrUpdateBlockReturnErr = err } -func (mbr *MockBlockRepository) SetMissingBlockNumbersReturnArray(returnArray []int64) { - mbr.missingBlockNumbersReturnArray = returnArray +func (repository *MockBlockRepository) SetMissingBlockNumbersReturnArray(returnArray []int64) { + repository.missingBlockNumbersReturnArray = returnArray } -func (mbr *MockBlockRepository) CreateOrUpdateBlock(block core.Block) (int64, error) { - mbr.createOrUpdateBlockCalled = true - mbr.createOrUpdateBlockPassedBlock = block - return mbr.createOrUpdateBlockReturnInt, mbr.createOrUpdateBlockReturnErr +func (repository *MockBlockRepository) CreateOrUpdateBlock(block core.Block) (int64, error) { + repository.createOrUpdateBlockCallCount++ + repository.createOrUpdateBlockCalled = true + repository.createOrUpdateBlockPassedBlock = block + repository.createOrUpdateBlockPassedBlockNumbers = append(repository.createOrUpdateBlockPassedBlockNumbers, block.Number) + return repository.createOrUpdateBlockReturnInt, repository.createOrUpdateBlockReturnErr } -func (mbr *MockBlockRepository) GetBlock(blockNumber int64) (core.Block, error) { - panic("implement me") +func (repository *MockBlockRepository) GetBlock(blockNumber int64) (core.Block, error) { + return core.Block{Number: blockNumber}, nil } -func (mbr *MockBlockRepository) MissingBlockNumbers(startingBlockNumber int64, endingBlockNumber int64, nodeId string) []int64 { - mbr.missingBlockNumbersCalled = true - mbr.missingBlockNumbersPassedStartingBlockNumber = startingBlockNumber - mbr.missingBlockNumbersPassedEndingBlockNumber = endingBlockNumber - mbr.missingBlockNumbersPassedNodeId = nodeId - return mbr.missingBlockNumbersReturnArray +func (repository *MockBlockRepository) MissingBlockNumbers(startingBlockNumber int64, endingBlockNumber int64, nodeId string) []int64 { + repository.missingBlockNumbersCalled = true + repository.missingBlockNumbersPassedStartingBlockNumber = startingBlockNumber + repository.missingBlockNumbersPassedEndingBlockNumber = endingBlockNumber + repository.missingBlockNumbersPassedNodeId = nodeId + return repository.missingBlockNumbersReturnArray } -func (mbr *MockBlockRepository) SetBlocksStatus(chainHead int64) { - mbr.setBlockStatusCalled = true - mbr.setBlockStatusPassedChainHead = chainHead +func (repository *MockBlockRepository) SetBlocksStatus(chainHead int64) { + repository.setBlockStatusCalled = true + repository.setBlockStatusPassedChainHead = chainHead } -func (mbr *MockBlockRepository) AssertCreateOrUpdateBlockCalledWith(block core.Block) { - Expect(mbr.createOrUpdateBlockCalled).To(BeTrue()) - Expect(mbr.createOrUpdateBlockPassedBlock).To(Equal(block)) +func (repository *MockBlockRepository) AssertCreateOrUpdateBlockCallCountEquals(times int) { + Expect(repository.createOrUpdateBlockCallCount).To(Equal(times)) } -func (mbr *MockBlockRepository) AssertMissingBlockNumbersCalledWith(startingBlockNumber int64, endingBlockNumber int64, nodeId string) { - Expect(mbr.missingBlockNumbersCalled).To(BeTrue()) - Expect(mbr.missingBlockNumbersPassedStartingBlockNumber).To(Equal(startingBlockNumber)) - Expect(mbr.missingBlockNumbersPassedEndingBlockNumber).To(Equal(endingBlockNumber)) - Expect(mbr.missingBlockNumbersPassedNodeId).To(Equal(nodeId)) +func (repository *MockBlockRepository) AssertCreateOrUpdateBlocksCallCountAndBlockNumbersEquals(times int, blockNumbers []int64) { + Expect(repository.createOrUpdateBlockCallCount).To(Equal(times)) + Expect(repository.createOrUpdateBlockPassedBlockNumbers).To(Equal(blockNumbers)) } -func (mbr *MockBlockRepository) AssertSetBlockStatusCalledWith(chainHead int64) { - Expect(mbr.setBlockStatusCalled).To(BeTrue()) - Expect(mbr.setBlockStatusPassedChainHead).To(Equal(chainHead)) +func (repository *MockBlockRepository) AssertCreateOrUpdateBlockCalledWith(block core.Block) { + Expect(repository.createOrUpdateBlockCalled).To(BeTrue()) + Expect(repository.createOrUpdateBlockPassedBlock).To(Equal(block)) +} + +func (repository *MockBlockRepository) AssertMissingBlockNumbersCalledWith(startingBlockNumber int64, endingBlockNumber int64, nodeId string) { + Expect(repository.missingBlockNumbersCalled).To(BeTrue()) + Expect(repository.missingBlockNumbersPassedStartingBlockNumber).To(Equal(startingBlockNumber)) + Expect(repository.missingBlockNumbersPassedEndingBlockNumber).To(Equal(endingBlockNumber)) + Expect(repository.missingBlockNumbersPassedNodeId).To(Equal(nodeId)) +} + +func (repository *MockBlockRepository) AssertSetBlockStatusCalledWith(chainHead int64) { + Expect(repository.setBlockStatusCalled).To(BeTrue()) + Expect(repository.setBlockStatusPassedChainHead).To(Equal(chainHead)) } diff --git a/pkg/fakes/mock_blockchain.go b/pkg/fakes/mock_blockchain.go new file mode 100644 index 00000000..bd543106 --- /dev/null +++ b/pkg/fakes/mock_blockchain.go @@ -0,0 +1,83 @@ +package fakes + +import ( + "math/big" + + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" +) + +type MockBlockChain struct { + fetchContractDataErr error + fetchContractDataPassedAbi string + fetchContractDataPassedAddress string + fetchContractDataPassedMethod string + fetchContractDataPassedMethodArg interface{} + fetchContractDataPassedResult interface{} + fetchContractDataPassedBlockNumber int64 + getBlockByNumberErr error + lastBlock *big.Int + node core.Node +} + +func NewMockBlockChain() *MockBlockChain { + return &MockBlockChain{ + node: core.Node{GenesisBlock: "GENESIS", NetworkID: 1, ID: "x123", ClientName: "Geth"}, + } +} + +func (blockChain *MockBlockChain) SetFetchContractDataErr(err error) { + blockChain.fetchContractDataErr = err +} + +func (blockChain *MockBlockChain) SetLastBlock(blockNumber *big.Int) { + blockChain.lastBlock = blockNumber +} + +func (blockChain *MockBlockChain) SetGetBlockByNumberErr(err error) { + blockChain.getBlockByNumberErr = err +} + +func (blockChain *MockBlockChain) GetHeaderByNumber(blockNumber int64) (core.Header, error) { + return core.Header{BlockNumber: blockNumber}, nil +} + +func (blockChain *MockBlockChain) FetchContractData(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) error { + blockChain.fetchContractDataPassedAbi = abiJSON + blockChain.fetchContractDataPassedAddress = address + blockChain.fetchContractDataPassedMethod = method + blockChain.fetchContractDataPassedMethodArg = methodArg + blockChain.fetchContractDataPassedResult = result + blockChain.fetchContractDataPassedBlockNumber = blockNumber + return blockChain.fetchContractDataErr +} + +func (blockChain *MockBlockChain) CallContract(contractHash string, input []byte, blockNumber *big.Int) ([]byte, error) { + return []byte{}, nil +} + +func (blockChain *MockBlockChain) LastBlock() *big.Int { + return blockChain.lastBlock +} + +func (blockChain *MockBlockChain) GetLogs(contract core.Contract, startingBlock *big.Int, endingBlock *big.Int) ([]core.Log, error) { + return []core.Log{}, nil +} + +func (blockChain *MockBlockChain) Node() core.Node { + return blockChain.node +} + +func (blockChain *MockBlockChain) GetBlockByNumber(blockNumber int64) (core.Block, error) { + return core.Block{Number: blockNumber}, blockChain.getBlockByNumberErr +} + +// TODO: handle methodArg being nil (can't match nil to nil in Gomega) +func (blockChain *MockBlockChain) AssertFetchContractDataCalledWith(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) { + Expect(blockChain.fetchContractDataPassedAbi).To(Equal(abiJSON)) + Expect(blockChain.fetchContractDataPassedAddress).To(Equal(address)) + Expect(blockChain.fetchContractDataPassedMethod).To(Equal(method)) + Expect(blockChain.fetchContractDataPassedResult).To(Equal(result)) + Expect(blockChain.fetchContractDataPassedBlockNumber).To(Equal(blockNumber)) +} diff --git a/pkg/fakes/mock_eth_client.go b/pkg/fakes/mock_eth_client.go new file mode 100644 index 00000000..c1e544d3 --- /dev/null +++ b/pkg/fakes/mock_eth_client.go @@ -0,0 +1,162 @@ +package fakes + +import ( + "context" + "math/big" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/gomega" +) + +type MockEthClient struct { + callContractErr error + callContractPassedContext context.Context + callContractPassedMsg ethereum.CallMsg + callContractPassedNumber *big.Int + callContractReturnBytes []byte + blockByNumberErr error + blockByNumberPassedContext context.Context + blockByNumberPassedNumber *big.Int + blockByNumberReturnBlock *types.Block + headerByNumberErr error + headerByNumberPassedContext context.Context + headerByNumberPassedNumber *big.Int + headerByNumberReturnHeader *types.Header + filterLogsErr error + filterLogsPassedContext context.Context + filterLogsPassedQuery ethereum.FilterQuery + filterLogsReturnLogs []types.Log + transactionReceipts map[string]*types.Receipt + err error + transactionSenderErr error + transactionReceiptErr error +} + +func NewMockEthClient() *MockEthClient { + return &MockEthClient{ + callContractErr: nil, + callContractPassedContext: nil, + callContractPassedMsg: ethereum.CallMsg{}, + callContractPassedNumber: nil, + callContractReturnBytes: nil, + blockByNumberErr: nil, + blockByNumberPassedContext: nil, + blockByNumberPassedNumber: nil, + blockByNumberReturnBlock: nil, + headerByNumberErr: nil, + headerByNumberPassedContext: nil, + headerByNumberPassedNumber: nil, + headerByNumberReturnHeader: nil, + filterLogsErr: nil, + filterLogsPassedContext: nil, + filterLogsPassedQuery: ethereum.FilterQuery{}, + filterLogsReturnLogs: nil, + transactionReceipts: make(map[string]*types.Receipt), + err: nil, + } +} + +func (client *MockEthClient) SetCallContractErr(err error) { + client.callContractErr = err +} + +func (client *MockEthClient) SetCallContractReturnBytes(returnBytes []byte) { + client.callContractReturnBytes = returnBytes +} + +func (client *MockEthClient) SetBlockByNumberErr(err error) { + client.blockByNumberErr = err +} + +func (client *MockEthClient) SetBlockByNumberReturnBlock(block *types.Block) { + client.blockByNumberReturnBlock = block +} + +func (client *MockEthClient) SetHeaderByNumberErr(err error) { + client.headerByNumberErr = err +} + +func (client *MockEthClient) SetHeaderByNumberReturnHeader(header *types.Header) { + client.headerByNumberReturnHeader = header +} + +func (client *MockEthClient) SetFilterLogsErr(err error) { + client.filterLogsErr = err +} + +func (client *MockEthClient) SetFilterLogsReturnLogs(logs []types.Log) { + client.filterLogsReturnLogs = logs +} + +func (client *MockEthClient) SetTransactionReceiptErr(err error) { + client.transactionReceiptErr = err +} + +func (client *MockEthClient) SetTransactionReceipts(receipts []*types.Receipt) { + for _, receipt := range receipts { + client.transactionReceipts[receipt.TxHash.Hex()] = receipt + } +} + +func (client *MockEthClient) SetTransactionSenderErr(err error) { + client.transactionSenderErr = err +} + +func (client *MockEthClient) CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) { + client.callContractPassedContext = ctx + client.callContractPassedMsg = msg + client.callContractPassedNumber = blockNumber + return client.callContractReturnBytes, client.callContractErr +} + +func (client *MockEthClient) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) { + client.blockByNumberPassedContext = ctx + client.blockByNumberPassedNumber = number + return client.blockByNumberReturnBlock, client.blockByNumberErr +} + +func (client *MockEthClient) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) { + client.headerByNumberPassedContext = ctx + client.headerByNumberPassedNumber = number + return client.headerByNumberReturnHeader, client.headerByNumberErr +} + +func (client *MockEthClient) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) { + client.filterLogsPassedContext = ctx + client.filterLogsPassedQuery = q + return client.filterLogsReturnLogs, client.filterLogsErr +} + +func (client *MockEthClient) TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) { + return common.HexToAddress("0x123"), client.transactionSenderErr +} + +func (client *MockEthClient) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { + if gasUsed, ok := client.transactionReceipts[txHash.Hex()]; ok { + return gasUsed, client.transactionReceiptErr + } + return &types.Receipt{GasUsed: uint64(0)}, client.transactionReceiptErr +} + +func (client *MockEthClient) AssertCallContractCalledWith(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) { + Expect(client.callContractPassedContext).To(Equal(ctx)) + Expect(client.callContractPassedMsg).To(Equal(msg)) + Expect(client.callContractPassedNumber).To(Equal(blockNumber)) +} + +func (client *MockEthClient) AssertBlockByNumberCalledWith(ctx context.Context, number *big.Int) { + Expect(client.blockByNumberPassedContext).To(Equal(ctx)) + Expect(client.blockByNumberPassedNumber).To(Equal(number)) +} + +func (client *MockEthClient) AssertHeaderByNumberCalledWith(ctx context.Context, number *big.Int) { + Expect(client.headerByNumberPassedContext).To(Equal(ctx)) + Expect(client.headerByNumberPassedNumber).To(Equal(number)) +} + +func (client *MockEthClient) AssertFilterLogsCalledWith(ctx context.Context, q ethereum.FilterQuery) { + Expect(client.filterLogsPassedContext).To(Equal(ctx)) + Expect(client.filterLogsPassedQuery).To(Equal(q)) +} diff --git a/pkg/fakes/mock_header_repository.go b/pkg/fakes/mock_header_repository.go new file mode 100644 index 00000000..e5254e9c --- /dev/null +++ b/pkg/fakes/mock_header_repository.go @@ -0,0 +1,40 @@ +package fakes + +import ( + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" +) + +type MockHeaderRepository struct { + createOrUpdateBlockNumbersCallCount int + createOrUpdateBlockNumbersPassedBlockNumbers []int64 + missingBlockNumbers []int64 +} + +func NewMockHeaderRepository() *MockHeaderRepository { + return &MockHeaderRepository{} +} + +func (repository *MockHeaderRepository) SetMissingBlockNumbers(blockNumbers []int64) { + repository.missingBlockNumbers = blockNumbers +} + +func (repository *MockHeaderRepository) CreateOrUpdateHeader(header core.Header) (int64, error) { + repository.createOrUpdateBlockNumbersCallCount++ + repository.createOrUpdateBlockNumbersPassedBlockNumbers = append(repository.createOrUpdateBlockNumbersPassedBlockNumbers, header.BlockNumber) + return 0, nil +} + +func (*MockHeaderRepository) GetHeader(blockNumber int64) (core.Header, error) { + return core.Header{BlockNumber: blockNumber}, nil +} + +func (repository *MockHeaderRepository) MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) []int64 { + return repository.missingBlockNumbers +} + +func (repository *MockHeaderRepository) AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(times int, blockNumbers []int64) { + Expect(repository.createOrUpdateBlockNumbersCallCount).To(Equal(times)) + Expect(repository.createOrUpdateBlockNumbersPassedBlockNumbers).To(Equal(blockNumbers)) +} diff --git a/pkg/fakes/mock_rpc_client.go b/pkg/fakes/mock_rpc_client.go new file mode 100644 index 00000000..d4cbf9b1 --- /dev/null +++ b/pkg/fakes/mock_rpc_client.go @@ -0,0 +1,73 @@ +package fakes + +import ( + "context" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/p2p" + "github.com/vulcanize/vulcanizedb/pkg/core" +) + +type MockRpcClient struct { + ipcPath string + nodeType core.NodeType +} + +func NewMockRpcClient() *MockRpcClient { + return &MockRpcClient{} +} + +func (client *MockRpcClient) SetNodeType(nodeType core.NodeType) { + client.nodeType = nodeType +} + +func (client *MockRpcClient) SetIpcPath(ipcPath string) { + client.ipcPath = ipcPath +} + +func (*MockRpcClient) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error { + switch method { + case "admin_nodeInfo": + if p, ok := result.(*p2p.NodeInfo); ok { + p.ID = "enode://GethNode@172.17.0.1:30303" + p.Name = "Geth/v1.7" + } + case "eth_getBlockByNumber": + if p, ok := result.(*types.Header); ok { + *p = types.Header{} + } + + case "parity_versionInfo": + if p, ok := result.(*core.ParityNodeInfo); ok { + *p = core.ParityNodeInfo{ + Track: "", + ParityVersion: core.ParityVersion{ + Major: 1, + Minor: 2, + Patch: 3, + }, + Hash: "", + } + } + case "parity_enode": + if p, ok := result.(*string); ok { + *p = "enode://ParityNode@172.17.0.1:30303" + } + case "net_version": + if p, ok := result.(*string); ok { + *p = "1234" + } + } + return nil +} + +func (client *MockRpcClient) IpcPath() string { + return client.ipcPath +} + +func (client *MockRpcClient) SupportedModules() (map[string]string, error) { + result := make(map[string]string) + if client.nodeType == core.GETH { + result["admin"] = "ok" + } + return result, nil +} diff --git a/pkg/geth/blockchain.go b/pkg/geth/blockchain.go index 0d10b7c3..a6d9193d 100644 --- a/pkg/geth/blockchain.go +++ b/pkg/geth/blockchain.go @@ -3,46 +3,47 @@ package geth import ( "math/big" - "log" - "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethclient" - "github.com/ethereum/go-ethereum/rpc" "golang.org/x/net/context" "github.com/vulcanize/vulcanizedb/pkg/core" vulcCommon "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common" - vulcRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" - "github.com/vulcanize/vulcanizedb/pkg/geth/node" ) -type Blockchain struct { - client *ethclient.Client - blockConverter vulcCommon.BlockConverter - readGethHeaders chan *types.Header - outputBlocks chan core.Block - newHeadSubscription ethereum.Subscription - node core.Node +type BlockChain struct { + client core.EthClient + blockConverter vulcCommon.BlockConverter + headerConverter vulcCommon.HeaderConverter + node core.Node } -func NewBlockchain(ipcPath string) *Blockchain { - blockchain := Blockchain{} - rpcClient, err := rpc.Dial(ipcPath) - if err != nil { - log.Fatal(err) +func NewBlockChain(client core.EthClient, node core.Node, converter vulcCommon.TransactionConverter) *BlockChain { + return &BlockChain{ + client: client, + blockConverter: vulcCommon.NewBlockConverter(converter), + headerConverter: vulcCommon.HeaderConverter{}, + node: node, } - client := ethclient.NewClient(rpcClient) - clientWrapper := node.ClientWrapper{ContextCaller: rpcClient, IPCPath: ipcPath} - blockchain.node = node.MakeNode(clientWrapper) - blockchain.client = client - transactionConverter := vulcRpc.NewRpcTransactionConverter(client) - blockchain.blockConverter = vulcCommon.NewBlockConverter(transactionConverter) - return &blockchain } -func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]core.Log, error) { +func (blockChain *BlockChain) GetBlockByNumber(blockNumber int64) (block core.Block, err error) { + gethBlock, err := blockChain.client.BlockByNumber(context.Background(), big.NewInt(blockNumber)) + if err != nil { + return block, err + } + return blockChain.blockConverter.ToCoreBlock(gethBlock) +} + +func (blockChain *BlockChain) GetHeaderByNumber(blockNumber int64) (header core.Header, err error) { + gethHeader, err := blockChain.client.HeaderByNumber(context.Background(), big.NewInt(blockNumber)) + if err != nil { + return header, err + } + return blockChain.headerConverter.Convert(gethHeader) +} + +func (blockChain *BlockChain) GetLogs(contract core.Contract, startingBlockNumber, endingBlockNumber *big.Int) ([]core.Log, error) { if endingBlockNumber == nil { endingBlockNumber = startingBlockNumber } @@ -52,7 +53,7 @@ func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumbe ToBlock: endingBlockNumber, Addresses: []common.Address{contractAddress}, } - gethLogs, err := blockchain.client.FilterLogs(context.Background(), fc) + gethLogs, err := blockChain.client.FilterLogs(context.Background(), fc) if err != nil { return []core.Log{}, err } @@ -60,23 +61,11 @@ func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumbe return logs, nil } -func (blockchain *Blockchain) Node() core.Node { - return blockchain.node -} - -func (blockchain *Blockchain) GetBlockByNumber(blockNumber int64) (core.Block, error) { - gethBlock, err := blockchain.client.BlockByNumber(context.Background(), big.NewInt(blockNumber)) - if err != nil { - return core.Block{}, err - } - block, err := blockchain.blockConverter.ToCoreBlock(gethBlock) - if err != nil { - return core.Block{}, err - } - return block, nil -} - -func (blockchain *Blockchain) LastBlock() *big.Int { - block, _ := blockchain.client.HeaderByNumber(context.Background(), nil) +func (blockChain *BlockChain) LastBlock() *big.Int { + block, _ := blockChain.client.HeaderByNumber(context.Background(), nil) return block.Number } + +func (blockChain *BlockChain) Node() core.Node { + return blockChain.node +} diff --git a/pkg/geth/blockchain_test.go b/pkg/geth/blockchain_test.go new file mode 100644 index 00000000..69f04e3b --- /dev/null +++ b/pkg/geth/blockchain_test.go @@ -0,0 +1,125 @@ +package geth_test + +import ( + "context" + "math/big" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + vulcCore "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/geth" + "github.com/vulcanize/vulcanizedb/pkg/geth/converters/cold_db" +) + +var _ = Describe("Geth blockchain", func() { + Describe("getting a block", func() { + It("fetches block from client", func() { + mockClient := fakes.NewMockEthClient() + mockClient.SetBlockByNumberReturnBlock(types.NewBlockWithHeader(&types.Header{})) + node := vulcCore.Node{} + blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter()) + blockNumber := int64(100) + + _, err := blockChain.GetBlockByNumber(blockNumber) + + Expect(err).NotTo(HaveOccurred()) + mockClient.AssertBlockByNumberCalledWith(context.Background(), big.NewInt(blockNumber)) + }) + + It("returns err if client returns err", func() { + mockClient := fakes.NewMockEthClient() + mockClient.SetBlockByNumberErr(fakes.FakeError) + node := vulcCore.Node{} + blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter()) + + _, err := blockChain.GetBlockByNumber(100) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + }) + + Describe("getting a header", func() { + It("fetches header from client", func() { + mockClient := fakes.NewMockEthClient() + blockNumber := int64(100) + mockClient.SetHeaderByNumberReturnHeader(&types.Header{Number: big.NewInt(blockNumber)}) + node := vulcCore.Node{} + blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter()) + + _, err := blockChain.GetHeaderByNumber(blockNumber) + + Expect(err).NotTo(HaveOccurred()) + mockClient.AssertHeaderByNumberCalledWith(context.Background(), big.NewInt(blockNumber)) + }) + + It("returns err if client returns err", func() { + mockClient := fakes.NewMockEthClient() + mockClient.SetHeaderByNumberErr(fakes.FakeError) + node := vulcCore.Node{} + blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter()) + + _, err := blockChain.GetHeaderByNumber(100) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + }) + + Describe("getting logs", func() { + It("fetches logs from client", func() { + mockClient := fakes.NewMockEthClient() + mockClient.SetFilterLogsReturnLogs([]types.Log{{}}) + node := vulcCore.Node{} + blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter()) + contract := vulcCore.Contract{Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex()} + startingBlockNumber := big.NewInt(1) + endingBlockNumber := big.NewInt(2) + + _, err := blockChain.GetLogs(contract, startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + expectedQuery := ethereum.FilterQuery{ + FromBlock: startingBlockNumber, + ToBlock: endingBlockNumber, + Addresses: []common.Address{common.HexToAddress(contract.Hash)}, + } + mockClient.AssertFilterLogsCalledWith(context.Background(), expectedQuery) + }) + + It("returns err if client returns err", func() { + mockClient := fakes.NewMockEthClient() + mockClient.SetFilterLogsErr(fakes.FakeError) + node := vulcCore.Node{} + blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter()) + contract := vulcCore.Contract{Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex()} + startingBlockNumber := big.NewInt(1) + endingBlockNumber := big.NewInt(2) + + _, err := blockChain.GetLogs(contract, startingBlockNumber, endingBlockNumber) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + }) + + Describe("getting the most recent block number", func() { + It("fetches latest header from client", func() { + mockClient := fakes.NewMockEthClient() + blockNumber := int64(100) + mockClient.SetHeaderByNumberReturnHeader(&types.Header{Number: big.NewInt(blockNumber)}) + node := vulcCore.Node{} + blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter()) + + result := blockChain.LastBlock() + + mockClient.AssertHeaderByNumberCalledWith(context.Background(), nil) + Expect(result).To(Equal(big.NewInt(blockNumber))) + }) + }) +}) diff --git a/pkg/geth/client/eth_client.go b/pkg/geth/client/eth_client.go new file mode 100644 index 00000000..21b7cd3d --- /dev/null +++ b/pkg/geth/client/eth_client.go @@ -0,0 +1,42 @@ +package client + +import ( + "context" + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + "math/big" +) + +type EthClient struct { + client *ethclient.Client +} + +func NewEthClient(client *ethclient.Client) EthClient { + return EthClient{client: client} +} + +func (client EthClient) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) { + return client.client.BlockByNumber(ctx, number) +} + +func (client EthClient) CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) { + return client.client.CallContract(ctx, msg, blockNumber) +} + +func (client EthClient) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) { + return client.client.FilterLogs(ctx, q) +} + +func (client EthClient) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) { + return client.client.HeaderByNumber(ctx, number) +} + +func (client EthClient) TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) { + return client.client.TransactionSender(ctx, tx, block, index) +} + +func (client EthClient) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { + return client.client.TransactionReceipt(ctx, txHash) +} diff --git a/pkg/geth/client/rpc_client.go b/pkg/geth/client/rpc_client.go new file mode 100644 index 00000000..46b3f80f --- /dev/null +++ b/pkg/geth/client/rpc_client.go @@ -0,0 +1,30 @@ +package client + +import ( + "context" + "github.com/ethereum/go-ethereum/rpc" +) + +type RpcClient struct { + client *rpc.Client + ipcPath string +} + +func NewRpcClient(client *rpc.Client, ipcPath string) RpcClient { + return RpcClient{ + client: client, + ipcPath: ipcPath, + } +} + +func (client RpcClient) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error { + return client.client.CallContext(ctx, result, method, args) +} + +func (client RpcClient) IpcPath() string { + return client.ipcPath +} + +func (client RpcClient) SupportedModules() (map[string]string, error) { + return client.client.SupportedModules() +} diff --git a/pkg/geth/contract.go b/pkg/geth/contract.go index 8813f545..f91b7ee4 100644 --- a/pkg/geth/contract.go +++ b/pkg/geth/contract.go @@ -14,7 +14,7 @@ var ( ErrInvalidStateAttribute = errors.New("invalid state attribute") ) -func (blockchain *Blockchain) FetchContractData(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) error { +func (blockChain *BlockChain) FetchContractData(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) error { parsed, err := ParseAbi(abiJSON) if err != nil { return err @@ -28,15 +28,15 @@ func (blockchain *Blockchain) FetchContractData(abiJSON string, address string, if err != nil { return err } - output, err := blockchain.callContract(address, input, big.NewInt(blockNumber)) + output, err := blockChain.callContract(address, input, big.NewInt(blockNumber)) if err != nil { return err } return parsed.Unpack(result, method, output) } -func (blockchain *Blockchain) callContract(contractHash string, input []byte, blockNumber *big.Int) ([]byte, error) { +func (blockChain *BlockChain) callContract(contractHash string, input []byte, blockNumber *big.Int) ([]byte, error) { to := common.HexToAddress(contractHash) msg := ethereum.CallMsg{To: &to, Data: input} - return blockchain.client.CallContract(context.Background(), msg, blockNumber) + return blockChain.client.CallContract(context.Background(), msg, blockNumber) } diff --git a/pkg/geth/converters/common/block_converter.go b/pkg/geth/converters/common/block_converter.go index 1f7699f5..6e13f474 100644 --- a/pkg/geth/converters/common/block_converter.go +++ b/pkg/geth/converters/common/block_converter.go @@ -20,6 +20,7 @@ func (bc BlockConverter) ToCoreBlock(gethBlock *types.Block) (core.Block, error) if err != nil { return core.Block{}, err } + coreBlock := core.Block{ Difficulty: gethBlock.Difficulty().Int64(), ExtraData: hexutil.Encode(gethBlock.Extra()), diff --git a/pkg/geth/converters/common/block_converter_test.go b/pkg/geth/converters/common/block_converter_test.go index 3e5d358c..ba05826b 100644 --- a/pkg/geth/converters/common/block_converter_test.go +++ b/pkg/geth/converters/common/block_converter_test.go @@ -1,8 +1,6 @@ package common_test import ( - "context" - "fmt" "io/ioutil" "log" "math/big" @@ -14,57 +12,11 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/fakes" vulcCommon "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common" "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" ) -type FakeGethClient struct { - receipts map[string]*types.Receipt - err error -} - -type TransActionReceiptError struct{} - -func (tarErr TransActionReceiptError) Error() string { - return fmt.Sprintf("transaction receipt error") -} - -type TransactionSenderError struct{} - -func (tasErr TransactionSenderError) Error() string { - return fmt.Sprintf("transaction sender error") -} - -func NewFakeClient(err error) *FakeGethClient { - return &FakeGethClient{ - receipts: make(map[string]*types.Receipt), - err: err, - } -} - -func (client *FakeGethClient) AddReceipts(receipts []*types.Receipt) { - for _, receipt := range receipts { - client.receipts[receipt.TxHash.Hex()] = receipt - } -} - -func (client *FakeGethClient) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { - if err, ok := client.err.(TransActionReceiptError); ok { - return &types.Receipt{}, err - } - if gasUsed, ok := client.receipts[txHash.Hex()]; ok { - return gasUsed, nil - } - return &types.Receipt{GasUsed: uint64(0)}, nil -} - -func (client *FakeGethClient) TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) { - if err, ok := client.err.(TransactionSenderError); ok { - return common.Address{}, err - } - return common.HexToAddress("0x123"), nil -} - var _ = Describe("Conversion of GethBlock to core.Block", func() { It("converts basic Block metadata", func() { @@ -90,7 +42,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() { UncleHash: common.Hash{128}, } block := types.NewBlock(&header, []*types.Transaction{}, []*types.Header{}, []*types.Receipt{}) - client := &FakeGethClient{} + client := fakes.NewMockEthClient() transactionConverter := rpc.NewRpcTransactionConverter(client) blockConverter := vulcCommon.NewBlockConverter(transactionConverter) @@ -132,8 +84,8 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() { } receipts := []*types.Receipt{&receipt} - client := NewFakeClient(nil) - client.AddReceipts(receipts) + client := fakes.NewMockEthClient() + client.SetTransactionReceipts(receipts) number := int64(1071819) header := types.Header{ @@ -176,8 +128,8 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() { } block := types.NewBlock(&header, transactions, uncles, receipts) - client := NewFakeClient(nil) - client.AddReceipts(receipts) + client := fakes.NewMockEthClient() + client.SetTransactionReceipts(receipts) transactionConverter := rpc.NewRpcTransactionConverter(client) blockConverter := vulcCommon.NewBlockConverter(transactionConverter) @@ -225,8 +177,8 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() { var uncles []*types.Header block := types.NewBlock(&header, transactions, uncles, receipts) - client := NewFakeClient(nil) - client.AddReceipts(receipts) + client := fakes.NewMockEthClient() + client.SetTransactionReceipts(receipts) transactionConverter := rpc.NewRpcTransactionConverter(client) blockConverter := vulcCommon.NewBlockConverter(transactionConverter) @@ -241,7 +193,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() { It("is empty", func() { header := types.Header{} block := types.NewBlock(&header, []*types.Transaction{}, []*types.Header{}, []*types.Receipt{}) - client := &FakeGethClient{} + client := fakes.NewMockEthClient() transactionConverter := rpc.NewRpcTransactionConverter(client) blockConverter := vulcCommon.NewBlockConverter(transactionConverter) @@ -270,8 +222,8 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() { TxHash: gethTransaction.Hash(), } - client := NewFakeClient(nil) - client.AddReceipts([]*types.Receipt{gethReceipt}) + client := fakes.NewMockEthClient() + client.SetTransactionReceipts([]*types.Receipt{gethReceipt}) header := types.Header{} block := types.NewBlock( @@ -317,8 +269,8 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() { ContractAddress: common.HexToAddress("0x1023342345"), } - client := NewFakeClient(nil) - client.AddReceipts([]*types.Receipt{gethReceipt}) + client := fakes.NewMockEthClient() + client.SetTransactionReceipts([]*types.Receipt{gethReceipt}) block := types.NewBlock( &types.Header{}, @@ -373,25 +325,25 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() { }) It("returns an error when transaction sender call fails", func() { - client := NewFakeClient(TransactionSenderError{}) - client.AddReceipts([]*types.Receipt{}) + client := fakes.NewMockEthClient() + client.SetTransactionSenderErr(fakes.FakeError) transactionConverter := rpc.NewRpcTransactionConverter(client) blockConverter := vulcCommon.NewBlockConverter(transactionConverter) _, err := blockConverter.ToCoreBlock(block) - Expect(err).To(Equal(TransactionSenderError{})) + Expect(err).To(MatchError(fakes.FakeError)) }) It("returns an error when transaction receipt call fails", func() { - client := NewFakeClient(TransActionReceiptError{}) - client.AddReceipts([]*types.Receipt{}) + client := fakes.NewMockEthClient() + client.SetTransactionReceiptErr(fakes.FakeError) transactionConverter := rpc.NewRpcTransactionConverter(client) blockConverter := vulcCommon.NewBlockConverter(transactionConverter) _, err := blockConverter.ToCoreBlock(block) - Expect(err).To(Equal(TransActionReceiptError{})) + Expect(err).To(MatchError(fakes.FakeError)) }) }) diff --git a/pkg/geth/converters/common/header_converter.go b/pkg/geth/converters/common/header_converter.go new file mode 100644 index 00000000..0638f75d --- /dev/null +++ b/pkg/geth/converters/common/header_converter.go @@ -0,0 +1,24 @@ +package common + +import ( + "bytes" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/vulcanize/vulcanizedb/pkg/core" +) + +type HeaderConverter struct{} + +func (converter HeaderConverter) Convert(gethHeader *types.Header) (core.Header, error) { + writer := new(bytes.Buffer) + err := rlp.Encode(writer, &gethHeader) + if err != nil { + panic(err) + } + coreHeader := core.Header{ + Hash: gethHeader.Hash().Hex(), + BlockNumber: gethHeader.Number.Int64(), + Raw: writer.Bytes(), + } + return coreHeader, nil +} diff --git a/pkg/geth/converters/common/header_converter_test.go b/pkg/geth/converters/common/header_converter_test.go new file mode 100644 index 00000000..76ef4fcc --- /dev/null +++ b/pkg/geth/converters/common/header_converter_test.go @@ -0,0 +1,47 @@ +package common_test + +import ( + "bytes" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + common2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common" + "math/big" +) + +var _ = Describe("Block header converter", func() { + It("converts geth header to core header", func() { + gethHeader := &types.Header{ + Difficulty: big.NewInt(1), + Number: big.NewInt(2), + ParentHash: common.HexToHash("0xParent"), + ReceiptHash: common.HexToHash("0xReceipt"), + Root: common.HexToHash("0xRoot"), + Time: big.NewInt(3), + TxHash: common.HexToHash("0xTransaction"), + UncleHash: common.HexToHash("0xUncle"), + } + converter := common2.HeaderConverter{} + + coreHeader, err := converter.Convert(gethHeader) + + Expect(err).NotTo(HaveOccurred()) + Expect(coreHeader.BlockNumber).To(Equal(gethHeader.Number.Int64())) + Expect(coreHeader.Hash).To(Equal(gethHeader.Hash().Hex())) + }) + + It("includes raw bytes for header", func() { + headerRLP := []byte{249, 2, 23, 160, 180, 251, 173, 248, 234, 69, 43, 19, 151, 24, 226, 112, 13, 193, 19, 92, 252, 129, 20, 80, 49, 200, 75, 122, 178, 124, 215, 16, 57, 79, 123, 56, 160, 29, 204, 77, 232, 222, 199, 93, 122, 171, 133, 181, 103, 182, 204, 212, 26, 211, 18, 69, 27, 148, 138, 116, 19, 240, 161, 66, 253, 64, 212, 147, 71, 148, 42, 101, 172, 164, 213, 252, 91, 92, 133, 144, 144, 166, 195, 77, 22, 65, 53, 57, 130, 38, 160, 14, 6, 111, 60, 34, 151, 165, 203, 48, 5, 147, 5, 38, 23, 209, 188, 165, 148, 111, 12, 170, 6, 53, 253, 177, 184, 90, 199, 229, 35, 111, 52, 160, 101, 186, 136, 127, 203, 8, 38, 246, 22, 208, 31, 115, 108, 29, 45, 103, 123, 202, 189, 226, 247, 252, 37, 170, 145, 207, 188, 11, 59, 173, 92, 179, 160, 32, 227, 83, 69, 64, 202, 241, 99, 120, 230, 232, 106, 43, 241, 35, 109, 159, 135, 109, 50, 24, 251, 192, 57, 88, 230, 219, 28, 99, 75, 35, 51, 185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 134, 11, 105, 222, 129, 162, 43, 131, 15, 66, 64, 131, 47, 239, 216, 130, 196, 68, 132, 86, 191, 180, 21, 152, 215, 131, 1, 3, 3, 132, 71, 101, 116, 104, 135, 103, 111, 49, 46, 53, 46, 49, 133, 108, 105, 110, 117, 120, 160, 146, 196, 18, 154, 10, 226, 54, 27, 69, 42, 158, 222, 236, 229, 92, 18, 236, 238, 171, 134, 99, 22, 25, 94, 61, 135, 252, 27, 0, 91, 102, 69, 136, 205, 76, 85, 185, 65, 207, 144, 21} + var gethHeader types.Header + err := rlp.Decode(bytes.NewReader(headerRLP), &gethHeader) + Expect(err).NotTo(HaveOccurred()) + converter := common2.HeaderConverter{} + + coreHeader, err := converter.Convert(&gethHeader) + + Expect(err).NotTo(HaveOccurred()) + Expect(coreHeader.Raw).To(Equal(headerRLP)) + }) +}) diff --git a/pkg/geth/converters/rpc/transaction_converter.go b/pkg/geth/converters/rpc/transaction_converter.go index 3bc0988e..c1dda565 100644 --- a/pkg/geth/converters/rpc/transaction_converter.go +++ b/pkg/geth/converters/rpc/transaction_converter.go @@ -14,16 +14,11 @@ import ( vulcCommon "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common" ) -type Client interface { - TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) - TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) -} - type RpcTransactionConverter struct { - client Client + client core.EthClient } -func NewRpcTransactionConverter(client Client) *RpcTransactionConverter { +func NewRpcTransactionConverter(client core.EthClient) *RpcTransactionConverter { return &RpcTransactionConverter{client: client} } @@ -42,7 +37,7 @@ func (rtc *RpcTransactionConverter) ConvertTransactionsToCore(gethBlock *types.B return err } coreTransaction := transToCoreTrans(transaction, &from) - coreTransaction, err = appendReceiptToTransaction(rtc.client, coreTransaction) + coreTransaction, err = rtc.appendReceiptToTransaction(coreTransaction) if err != nil { log.Println("receipt: ", err) return err @@ -58,8 +53,8 @@ func (rtc *RpcTransactionConverter) ConvertTransactionsToCore(gethBlock *types.B return coreTransactions, nil } -func appendReceiptToTransaction(client Client, transaction core.Transaction) (core.Transaction, error) { - gethReceipt, err := client.TransactionReceipt(context.Background(), common.HexToHash(transaction.Hash)) +func (rtc *RpcTransactionConverter) appendReceiptToTransaction(transaction core.Transaction) (core.Transaction, error) { + gethReceipt, err := rtc.client.TransactionReceipt(context.Background(), common.HexToHash(transaction.Hash)) if err != nil { return transaction, err } diff --git a/pkg/geth/node/node.go b/pkg/geth/node/node.go index daa20e76..33459e8f 100644 --- a/pkg/geth/node/node.go +++ b/pkg/geth/node/node.go @@ -7,69 +7,38 @@ import ( "regexp" - "strings" - "log" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/p2p" "github.com/vulcanize/vulcanizedb/pkg/core" + "strings" ) -type PropertiesReader interface { +type IPropertiesReader interface { NodeInfo() (id string, name string) NetworkId() float64 GenesisBlock() string } -type ClientWrapper struct { - ContextCaller - IPCPath string -} - -type ContextCaller interface { - CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error - SupportedModules() (map[string]string, error) +type PropertiesReader struct { + client core.RpcClient } type ParityClient struct { - ClientWrapper + PropertiesReader } type GethClient struct { - ClientWrapper + PropertiesReader } type InfuraClient struct { - ClientWrapper + PropertiesReader } -func (clientWrapper ClientWrapper) NodeType() core.NodeType { - if strings.Contains(clientWrapper.IPCPath, "infura") { - return core.INFURA - } - modules, _ := clientWrapper.SupportedModules() - if _, ok := modules["admin"]; ok { - return core.GETH - } - return core.PARITY -} - -func makePropertiesReader(wrapper ClientWrapper) PropertiesReader { - switch wrapper.NodeType() { - case core.GETH: - return GethClient{ClientWrapper: wrapper} - case core.PARITY: - return ParityClient{ClientWrapper: wrapper} - case core.INFURA: - return InfuraClient{ClientWrapper: wrapper} - default: - return wrapper - } -} - -func MakeNode(wrapper ClientWrapper) core.Node { - pr := makePropertiesReader(wrapper) +func MakeNode(rpcClient core.RpcClient) core.Node { + pr := makePropertiesReader(rpcClient) id, name := pr.NodeInfo() return core.Node{ GenesisBlock: pr.GenesisBlock(), @@ -79,9 +48,33 @@ func MakeNode(wrapper ClientWrapper) core.Node { } } -func (client ClientWrapper) NetworkId() float64 { +func makePropertiesReader(client core.RpcClient) IPropertiesReader { + switch getNodeType(client) { + case core.GETH: + return GethClient{PropertiesReader: PropertiesReader{client: client}} + case core.PARITY: + return ParityClient{PropertiesReader: PropertiesReader{client: client}} + case core.INFURA: + return InfuraClient{PropertiesReader: PropertiesReader{client: client}} + default: + return PropertiesReader{client: client} + } +} + +func getNodeType(client core.RpcClient) core.NodeType { + if strings.Contains(client.IpcPath(), "infura") { + return core.INFURA + } + modules, _ := client.SupportedModules() + if _, ok := modules["admin"]; ok { + return core.GETH + } + return core.PARITY +} + +func (reader PropertiesReader) NetworkId() float64 { var version string - err := client.CallContext(context.Background(), &version, "net_version") + err := reader.client.CallContext(context.Background(), &version, "net_version") if err != nil { log.Println(err) } @@ -89,17 +82,17 @@ func (client ClientWrapper) NetworkId() float64 { return networkId } -func (client ClientWrapper) GenesisBlock() string { +func (reader PropertiesReader) GenesisBlock() string { var header *types.Header blockZero := "0x0" includeTransactions := false - client.CallContext(context.Background(), &header, "eth_getBlockByNumber", blockZero, includeTransactions) + reader.client.CallContext(context.Background(), &header, "eth_getBlockByNumber", blockZero, includeTransactions) return header.Hash().Hex() } -func (client ClientWrapper) NodeInfo() (string, string) { +func (reader PropertiesReader) NodeInfo() (string, string) { var info p2p.NodeInfo - client.CallContext(context.Background(), &info, "admin_nodeInfo") + reader.client.CallContext(context.Background(), &info, "admin_nodeInfo") return info.ID, info.Name } @@ -115,14 +108,14 @@ func (client InfuraClient) NodeInfo() (string, string) { func (client ParityClient) parityNodeInfo() string { var nodeInfo core.ParityNodeInfo - client.CallContext(context.Background(), &nodeInfo, "parity_versionInfo") + client.client.CallContext(context.Background(), &nodeInfo, "parity_versionInfo") return nodeInfo.String() } func (client ParityClient) parityID() string { var enodeId = regexp.MustCompile(`^enode://(.+)@.+$`) var enodeURL string - client.CallContext(context.Background(), &enodeURL, "parity_enode") + client.client.CallContext(context.Background(), &enodeURL, "parity_enode") enode := enodeId.FindStringSubmatch(enodeURL) if len(enode) < 2 { return "" diff --git a/pkg/geth/node/node_test.go b/pkg/geth/node/node_test.go index 5fe9d593..ec557369 100644 --- a/pkg/geth/node/node_test.go +++ b/pkg/geth/node/node_test.go @@ -3,66 +3,15 @@ package node_test import ( "encoding/json" - "context" - - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/p2p" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/geth/node" ) -type MockContextCaller struct { - nodeType core.NodeType -} - var EmpytHeaderHash = "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" -func (MockContextCaller) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error { - switch method { - case "admin_nodeInfo": - if p, ok := result.(*p2p.NodeInfo); ok { - p.ID = "enode://GethNode@172.17.0.1:30303" - p.Name = "Geth/v1.7" - } - case "eth_getBlockByNumber": - if p, ok := result.(*types.Header); ok { - *p = types.Header{} - } - - case "parity_versionInfo": - if p, ok := result.(*core.ParityNodeInfo); ok { - *p = core.ParityNodeInfo{ - Track: "", - ParityVersion: core.ParityVersion{ - Major: 1, - Minor: 2, - Patch: 3, - }, - Hash: "", - } - } - case "parity_enode": - if p, ok := result.(*string); ok { - *p = "enode://ParityNode@172.17.0.1:30303" - } - case "net_version": - if p, ok := result.(*string); ok { - *p = "1234" - } - } - return nil -} - -func (mcc MockContextCaller) SupportedModules() (map[string]string, error) { - result := make(map[string]string) - if mcc.nodeType == core.GETH { - result["admin"] = "ok" - } - return result, nil -} - var _ = Describe("Parity Node Info", func() { It("verifies parity_versionInfo can be unmarshalled into ParityNodeInfo", func() { @@ -99,39 +48,38 @@ var _ = Describe("Parity Node Info", func() { }) It("returns the genesis block for any client", func() { - mcc := MockContextCaller{} - cw := node.ClientWrapper{ContextCaller: mcc} - n := node.MakeNode(cw) + client := fakes.NewMockRpcClient() + n := node.MakeNode(client) Expect(n.GenesisBlock).To(Equal(EmpytHeaderHash)) }) It("returns the network id for any client", func() { - mcc := MockContextCaller{} - cw := node.ClientWrapper{ContextCaller: mcc} - n := node.MakeNode(cw) + client := fakes.NewMockRpcClient() + n := node.MakeNode(client) Expect(n.NetworkID).To(Equal(float64(1234))) }) It("returns parity ID and client name for parity node", func() { - mcc := MockContextCaller{core.PARITY} - cw := node.ClientWrapper{ContextCaller: mcc} - n := node.MakeNode(cw) + client := fakes.NewMockRpcClient() + client.SetNodeType(core.PARITY) + n := node.MakeNode(client) Expect(n.ID).To(Equal("ParityNode")) Expect(n.ClientName).To(Equal("Parity/v1.2.3/")) }) It("returns geth ID and client name for geth node", func() { - mcc := MockContextCaller{core.GETH} - cw := node.ClientWrapper{ContextCaller: mcc} - n := node.MakeNode(cw) + client := fakes.NewMockRpcClient() + client.SetNodeType(core.GETH) + n := node.MakeNode(client) Expect(n.ID).To(Equal("enode://GethNode@172.17.0.1:30303")) Expect(n.ClientName).To(Equal("Geth/v1.7")) }) It("returns infura ID and client name for infura node", func() { - mcc := MockContextCaller{core.INFURA} - cw := node.ClientWrapper{ContextCaller: mcc, IPCPath: "https://mainnet.infura.io/123"} - n := node.MakeNode(cw) + client := fakes.NewMockRpcClient() + client.SetNodeType(core.INFURA) + client.SetIpcPath("infura/path") + n := node.MakeNode(client) Expect(n.ID).To(Equal("infura")) Expect(n.ClientName).To(Equal("infura")) }) diff --git a/pkg/history/block_validator.go b/pkg/history/block_validator.go new file mode 100644 index 00000000..4f98cbca --- /dev/null +++ b/pkg/history/block_validator.go @@ -0,0 +1,29 @@ +package history + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore" +) + +type BlockValidator struct { + blockchain core.BlockChain + blockRepository datastore.BlockRepository + windowSize int +} + +func NewBlockValidator(blockchain core.BlockChain, blockRepository datastore.BlockRepository, windowSize int) *BlockValidator { + return &BlockValidator{ + blockchain: blockchain, + blockRepository: blockRepository, + windowSize: windowSize, + } +} + +func (bv BlockValidator) ValidateBlocks() ValidationWindow { + window := MakeValidationWindow(bv.blockchain, bv.windowSize) + blockNumbers := MakeRange(window.LowerBound, window.UpperBound) + RetrieveAndUpdateBlocks(bv.blockchain, bv.blockRepository, blockNumbers) + lastBlock := bv.blockchain.LastBlock().Int64() + bv.blockRepository.SetBlocksStatus(lastBlock) + return window +} diff --git a/pkg/history/block_validator_test.go b/pkg/history/block_validator_test.go new file mode 100644 index 00000000..e05e6501 --- /dev/null +++ b/pkg/history/block_validator_test.go @@ -0,0 +1,33 @@ +package history_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/history" + "math/big" +) + +var _ = Describe("Blocks validator", func() { + + It("calls create or update for all blocks within the window", func() { + blockChain := fakes.NewMockBlockChain() + blockChain.SetLastBlock(big.NewInt(7)) + blocksRepository := fakes.NewMockBlockRepository() + validator := history.NewBlockValidator(blockChain, blocksRepository, 2) + + window := validator.ValidateBlocks() + + Expect(window).To(Equal(history.ValidationWindow{LowerBound: 5, UpperBound: 7})) + blocksRepository.AssertCreateOrUpdateBlockCallCountEquals(3) + }) + + It("returns the number of largest block", func() { + blockChain := fakes.NewMockBlockChain() + blockChain.SetLastBlock(big.NewInt(3)) + maxBlockNumber := blockChain.LastBlock() + + Expect(maxBlockNumber.Int64()).To(Equal(int64(3))) + }) +}) diff --git a/pkg/history/header_validator.go b/pkg/history/header_validator.go new file mode 100644 index 00000000..b733aaee --- /dev/null +++ b/pkg/history/header_validator.go @@ -0,0 +1,27 @@ +package history + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore" +) + +type HeaderValidator struct { + blockChain core.BlockChain + headerRepository datastore.HeaderRepository + windowSize int +} + +func NewHeaderValidator(blockChain core.BlockChain, repository datastore.HeaderRepository, windowSize int) HeaderValidator { + return HeaderValidator{ + blockChain: blockChain, + headerRepository: repository, + windowSize: windowSize, + } +} + +func (validator HeaderValidator) ValidateHeaders() ValidationWindow { + window := MakeValidationWindow(validator.blockChain, validator.windowSize) + blockNumbers := MakeRange(window.LowerBound, window.UpperBound) + RetrieveAndUpdateHeaders(validator.blockChain, validator.headerRepository, blockNumbers) + return window +} diff --git a/pkg/history/header_validator_test.go b/pkg/history/header_validator_test.go new file mode 100644 index 00000000..379f6739 --- /dev/null +++ b/pkg/history/header_validator_test.go @@ -0,0 +1,22 @@ +package history_test + +import ( + . "github.com/onsi/ginkgo" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/history" + "math/big" +) + +var _ = Describe("Header validator", func() { + It("attempts to create every header in the validation window", func() { + headerRepository := fakes.NewMockHeaderRepository() + headerRepository.SetMissingBlockNumbers([]int64{}) + blockChain := fakes.NewMockBlockChain() + blockChain.SetLastBlock(big.NewInt(3)) + validator := history.NewHeaderValidator(blockChain, headerRepository, 2) + + validator.ValidateHeaders() + + headerRepository.AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(3, []int64{1, 2, 3}) + }) +}) diff --git a/pkg/history/populate_blocks.go b/pkg/history/populate_blocks.go index 52106a75..364f1150 100644 --- a/pkg/history/populate_blocks.go +++ b/pkg/history/populate_blocks.go @@ -7,16 +7,16 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore" ) -func PopulateMissingBlocks(blockchain core.Blockchain, blockRepository datastore.BlockRepository, startingBlockNumber int64) int { +func PopulateMissingBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, startingBlockNumber int64) int { lastBlock := blockchain.LastBlock().Int64() - blockRange := blockRepository.MissingBlockNumbers(startingBlockNumber, lastBlock-1, blockchain.Node().ID) + blockRange := blockRepository.MissingBlockNumbers(startingBlockNumber, lastBlock, blockchain.Node().ID) log.SetPrefix("") log.Printf("Backfilling %d blocks\n\n", len(blockRange)) RetrieveAndUpdateBlocks(blockchain, blockRepository, blockRange) return len(blockRange) } -func RetrieveAndUpdateBlocks(blockchain core.Blockchain, blockRepository datastore.BlockRepository, blockNumbers []int64) int { +func RetrieveAndUpdateBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, blockNumbers []int64) int { for _, blockNumber := range blockNumbers { block, err := blockchain.GetBlockByNumber(blockNumber) if err != nil { diff --git a/pkg/history/populate_blocks_test.go b/pkg/history/populate_blocks_test.go index a98c76aa..e80687a3 100644 --- a/pkg/history/populate_blocks_test.go +++ b/pkg/history/populate_blocks_test.go @@ -1,35 +1,26 @@ package history_test import ( - "errors" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/inmemory" "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/history" + "math/big" ) var _ = Describe("Populating blocks", func() { - var inMemory *inmemory.InMemory - var blockRepository *inmemory.BlockRepository + var blockRepository *fakes.MockBlockRepository BeforeEach(func() { - inMemory = inmemory.NewInMemory() - blockRepository = &inmemory.BlockRepository{InMemory: inMemory} + blockRepository = fakes.NewMockBlockRepository() }) - It("fills in the only missing block (Number 1)", func() { - blocks := []core.Block{ - {Number: 1}, - {Number: 2}, - } - blockchain := fakes.NewBlockchainWithBlocks(blocks) + It("fills in the only missing block (BlockNumber 1)", func() { + blockChain := fakes.NewMockBlockChain() + blockChain.SetLastBlock(big.NewInt(2)) + blockRepository.SetMissingBlockNumbersReturnArray([]int64{2}) - blockRepository.CreateOrUpdateBlock(core.Block{Number: 2}) - - blocksAdded := history.PopulateMissingBlocks(blockchain, blockRepository, 1) + blocksAdded := history.PopulateMissingBlocks(blockChain, blockRepository, 1) _, err := blockRepository.GetBlock(1) Expect(blocksAdded).To(Equal(1)) @@ -37,77 +28,41 @@ var _ = Describe("Populating blocks", func() { }) It("fills in the three missing blocks (Numbers: 5,8,10)", func() { - blockchain := fakes.NewBlockchainWithBlocks([]core.Block{ - {Number: 4}, - {Number: 5}, - {Number: 6}, - {Number: 7}, - {Number: 8}, - {Number: 9}, - {Number: 10}, - {Number: 11}, - {Number: 12}, - {Number: 13}, - }) - blockRepository.CreateOrUpdateBlock(core.Block{Number: 1}) - blockRepository.CreateOrUpdateBlock(core.Block{Number: 2}) - blockRepository.CreateOrUpdateBlock(core.Block{Number: 3}) - blockRepository.CreateOrUpdateBlock(core.Block{Number: 6}) - blockRepository.CreateOrUpdateBlock(core.Block{Number: 7}) - blockRepository.CreateOrUpdateBlock(core.Block{Number: 9}) - blockRepository.CreateOrUpdateBlock(core.Block{Number: 11}) - blockRepository.CreateOrUpdateBlock(core.Block{Number: 12}) + blockChain := fakes.NewMockBlockChain() + blockChain.SetLastBlock(big.NewInt(13)) + blockRepository.SetMissingBlockNumbersReturnArray([]int64{5, 8, 10}) - blocksAdded := history.PopulateMissingBlocks(blockchain, blockRepository, 5) + blocksAdded := history.PopulateMissingBlocks(blockChain, blockRepository, 5) Expect(blocksAdded).To(Equal(3)) - Expect(blockRepository.BlockCount()).To(Equal(11)) - _, err := blockRepository.GetBlock(4) - Expect(err).To(HaveOccurred()) - _, err = blockRepository.GetBlock(5) - Expect(err).ToNot(HaveOccurred()) - _, err = blockRepository.GetBlock(8) - Expect(err).ToNot(HaveOccurred()) - _, err = blockRepository.GetBlock(10) - Expect(err).ToNot(HaveOccurred()) - _, err = blockRepository.GetBlock(13) - Expect(err).To(HaveOccurred()) + blockRepository.AssertCreateOrUpdateBlocksCallCountAndBlockNumbersEquals(3, []int64{5, 8, 10}) }) It("returns the number of blocks created", func() { - blockchain := fakes.NewBlockchainWithBlocks([]core.Block{ - {Number: 4}, - {Number: 5}, - {Number: 6}, - }) - blockRepository.CreateOrUpdateBlock(core.Block{Number: 3}) - blockRepository.CreateOrUpdateBlock(core.Block{Number: 6}) + blockChain := fakes.NewMockBlockChain() + blockChain.SetLastBlock(big.NewInt(6)) + blockRepository.SetMissingBlockNumbersReturnArray([]int64{4, 5}) - numberOfBlocksCreated := history.PopulateMissingBlocks(blockchain, blockRepository, 3) + numberOfBlocksCreated := history.PopulateMissingBlocks(blockChain, blockRepository, 3) Expect(numberOfBlocksCreated).To(Equal(2)) }) It("updates the repository with a range of blocks w/in the range ", func() { - blockchain := fakes.NewBlockchainWithBlocks([]core.Block{ - {Number: 1}, - {Number: 2}, - {Number: 3}, - {Number: 4}, - {Number: 5}, - }) + blockChain := fakes.NewMockBlockChain() - history.RetrieveAndUpdateBlocks(blockchain, blockRepository, history.MakeRange(2, 5)) - Expect(blockRepository.BlockCount()).To(Equal(3)) - Expect(blockRepository.CreateOrUpdateBlockCallCount).To(Equal(3)) + history.RetrieveAndUpdateBlocks(blockChain, blockRepository, history.MakeRange(2, 5)) + + blockRepository.AssertCreateOrUpdateBlocksCallCountAndBlockNumbersEquals(4, []int64{2, 3, 4, 5}) }) It("does not call repository create block when there is an error", func() { - blockchain := fakes.NewBlockchain(errors.New("error getting block")) + blockChain := fakes.NewMockBlockChain() + blockChain.SetGetBlockByNumberErr(fakes.FakeError) blocks := history.MakeRange(1, 10) - history.RetrieveAndUpdateBlocks(blockchain, blockRepository, blocks) - Expect(blockRepository.BlockCount()).To(Equal(0)) - Expect(blockRepository.CreateOrUpdateBlockCallCount).To(Equal(0)) - }) + history.RetrieveAndUpdateBlocks(blockChain, blockRepository, blocks) + + blockRepository.AssertCreateOrUpdateBlockCallCountEquals(0) + }) }) diff --git a/pkg/history/populate_headers.go b/pkg/history/populate_headers.go new file mode 100644 index 00000000..1fc1a80d --- /dev/null +++ b/pkg/history/populate_headers.go @@ -0,0 +1,29 @@ +package history + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore" + "log" +) + +func PopulateMissingHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, startingBlockNumber int64) int { + lastBlock := blockchain.LastBlock().Int64() + blockRange := headerRepository.MissingBlockNumbers(startingBlockNumber, lastBlock, blockchain.Node().ID) + log.SetPrefix("") + log.Printf("Backfilling %d blocks\n\n", len(blockRange)) + RetrieveAndUpdateHeaders(blockchain, headerRepository, blockRange) + return len(blockRange) +} + +func RetrieveAndUpdateHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, blockNumbers []int64) int { + for _, blockNumber := range blockNumbers { + header, err := blockchain.GetHeaderByNumber(blockNumber) + if err != nil { + log.Printf("failed to retrieve block number: %d\n", blockNumber) + return 0 + } + // TODO: handle possible error here + headerRepository.CreateOrUpdateHeader(header) + } + return len(blockNumbers) +} diff --git a/pkg/history/populate_headers_test.go b/pkg/history/populate_headers_test.go new file mode 100644 index 00000000..5d3bf8b5 --- /dev/null +++ b/pkg/history/populate_headers_test.go @@ -0,0 +1,43 @@ +package history_test + +import ( + "math/big" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/history" +) + +var _ = Describe("Populating headers", func() { + + var headerRepository *fakes.MockHeaderRepository + + BeforeEach(func() { + headerRepository = fakes.NewMockHeaderRepository() + }) + + Describe("When 1 missing header", func() { + + It("returns number of headers added", func() { + blockChain := fakes.NewMockBlockChain() + blockChain.SetLastBlock(big.NewInt(2)) + headerRepository.SetMissingBlockNumbers([]int64{2}) + + headersAdded := history.PopulateMissingHeaders(blockChain, headerRepository, 1) + + Expect(headersAdded).To(Equal(1)) + }) + }) + + It("adds missing headers to the db", func() { + blockChain := fakes.NewMockBlockChain() + blockChain.SetLastBlock(big.NewInt(2)) + headerRepository.SetMissingBlockNumbers([]int64{2}) + + history.PopulateMissingHeaders(blockChain, headerRepository, 1) + + headerRepository.AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(1, []int64{2}) + }) +}) diff --git a/pkg/history/validate_blocks.go b/pkg/history/validate_blocks.go deleted file mode 100644 index 1f6ecf9e..00000000 --- a/pkg/history/validate_blocks.go +++ /dev/null @@ -1,68 +0,0 @@ -package history - -import ( - "io" - "text/template" - - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore" -) - -const WindowTemplate = `Validating Blocks -|{{.LowerBound}}|-- Validation Window --|{{.UpperBound}}| ({{.UpperBound}}:HEAD) - -` - -var ParsedWindowTemplate = *template.Must(template.New("window").Parse(WindowTemplate)) - -type BlockValidator struct { - blockchain core.Blockchain - blockRepository datastore.BlockRepository - windowSize int - parsedLoggingTemplate template.Template -} - -func NewBlockValidator(blockchain core.Blockchain, blockRepository datastore.BlockRepository, windowSize int) *BlockValidator { - return &BlockValidator{ - blockchain, - blockRepository, - windowSize, - ParsedWindowTemplate, - } -} - -func (bv BlockValidator) ValidateBlocks() ValidationWindow { - window := MakeValidationWindow(bv.blockchain, bv.windowSize) - blockNumbers := MakeRange(window.LowerBound, window.UpperBound) - RetrieveAndUpdateBlocks(bv.blockchain, bv.blockRepository, blockNumbers) - lastBlock := bv.blockchain.LastBlock().Int64() - bv.blockRepository.SetBlocksStatus(lastBlock) - return window -} - -func (bv BlockValidator) Log(out io.Writer, window ValidationWindow) { - bv.parsedLoggingTemplate.Execute(out, window) -} - -type ValidationWindow struct { - LowerBound int64 - UpperBound int64 -} - -func (window ValidationWindow) Size() int { - return int(window.UpperBound - window.LowerBound) -} - -func MakeValidationWindow(blockchain core.Blockchain, windowSize int) ValidationWindow { - upperBound := blockchain.LastBlock().Int64() - lowerBound := upperBound - int64(windowSize) - return ValidationWindow{lowerBound, upperBound} -} - -func MakeRange(min, max int64) []int64 { - a := make([]int64, max-min) - for i := range a { - a[i] = min + int64(i) - } - return a -} diff --git a/pkg/history/validate_blocks_test.go b/pkg/history/validate_blocks_test.go deleted file mode 100644 index fc92521e..00000000 --- a/pkg/history/validate_blocks_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package history_test - -import ( - "bytes" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/inmemory" - "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/history" -) - -var _ = Describe("Blocks validator", func() { - - It("creates a ValidationWindow equal to (HEAD-windowSize, HEAD)", func() { - blockchain := fakes.NewBlockchainWithBlocks([]core.Block{ - {Number: 1}, - {Number: 2}, - {Number: 3}, - {Number: 4}, - {Number: 5}, - }) - - validationWindow := history.MakeValidationWindow(blockchain, 2) - - Expect(validationWindow.LowerBound).To(Equal(int64(3))) - Expect(validationWindow.UpperBound).To(Equal(int64(5))) - }) - - It("returns the window size", func() { - window := history.ValidationWindow{LowerBound: 1, UpperBound: 3} - Expect(window.Size()).To(Equal(2)) - }) - - It("calls create or update for all blocks within the window", func() { - blockchain := fakes.NewBlockchainWithBlocks([]core.Block{ - {Number: 4}, - {Number: 5}, - {Number: 6}, - {Number: 7}, - }) - inMemoryDB := inmemory.NewInMemory() - blocksRepository := &inmemory.BlockRepository{InMemory: inMemoryDB} - - validator := history.NewBlockValidator(blockchain, blocksRepository, 2) - window := validator.ValidateBlocks() - Expect(window).To(Equal(history.ValidationWindow{LowerBound: 5, UpperBound: 7})) - Expect(blocksRepository.BlockCount()).To(Equal(2)) - Expect(blocksRepository.CreateOrUpdateBlockCallCount).To(Equal(2)) - }) - - It("logs window message", func() { - inMemoryDB := inmemory.NewInMemory() - blockRepository := &inmemory.BlockRepository{InMemory: inMemoryDB} - - expectedMessage := &bytes.Buffer{} - window := history.ValidationWindow{LowerBound: 5, UpperBound: 7} - history.ParsedWindowTemplate.Execute(expectedMessage, history.ValidationWindow{LowerBound: 5, UpperBound: 7}) - - blockchain := fakes.NewBlockchainWithBlocks([]core.Block{}) - validator := history.NewBlockValidator(blockchain, blockRepository, 2) - actualMessage := &bytes.Buffer{} - validator.Log(actualMessage, window) - Expect(actualMessage).To(Equal(expectedMessage)) - }) - - It("generates a range of int64s", func() { - numberOfBlocksCreated := history.MakeRange(0, 5) - expected := []int64{0, 1, 2, 3, 4} - - Expect(numberOfBlocksCreated).To(Equal(expected)) - }) - - It("returns the number of largest block", func() { - blockchain := fakes.NewBlockchainWithBlocks([]core.Block{ - {Number: 1}, - {Number: 2}, - {Number: 3}, - }) - maxBlockNumber := blockchain.LastBlock() - - Expect(maxBlockNumber.Int64()).To(Equal(int64(3))) - }) - -}) diff --git a/pkg/history/validation_window.go b/pkg/history/validation_window.go new file mode 100644 index 00000000..e10253c3 --- /dev/null +++ b/pkg/history/validation_window.go @@ -0,0 +1,41 @@ +package history + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "io" + "text/template" +) + +const WindowTemplate = `Validating Blocks +|{{.LowerBound}}|-- Validation Window --|{{.UpperBound}}| ({{.UpperBound}}:HEAD) + +` + +var ParsedWindowTemplate = *template.Must(template.New("window").Parse(WindowTemplate)) + +type ValidationWindow struct { + LowerBound int64 + UpperBound int64 +} + +func (window ValidationWindow) Size() int { + return int(window.UpperBound - window.LowerBound) +} + +func MakeValidationWindow(blockchain core.BlockChain, windowSize int) ValidationWindow { + upperBound := blockchain.LastBlock().Int64() + lowerBound := upperBound - int64(windowSize) + return ValidationWindow{lowerBound, upperBound} +} + +func MakeRange(min, max int64) []int64 { + a := make([]int64, max-min+1) + for i := range a { + a[i] = min + int64(i) + } + return a +} + +func (window ValidationWindow) Log(out io.Writer) { + ParsedWindowTemplate.Execute(out, window) +} diff --git a/pkg/history/validation_window_test.go b/pkg/history/validation_window_test.go new file mode 100644 index 00000000..354af33e --- /dev/null +++ b/pkg/history/validation_window_test.go @@ -0,0 +1,48 @@ +package history_test + +import ( + "bytes" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/history" + "math/big" +) + +var _ = Describe("Validation window", func() { + It("creates a ValidationWindow equal to (HEAD-windowSize, HEAD)", func() { + blockChain := fakes.NewMockBlockChain() + blockChain.SetLastBlock(big.NewInt(5)) + + validationWindow := history.MakeValidationWindow(blockChain, 2) + + Expect(validationWindow.LowerBound).To(Equal(int64(3))) + Expect(validationWindow.UpperBound).To(Equal(int64(5))) + }) + + It("returns the window size", func() { + window := history.ValidationWindow{LowerBound: 1, UpperBound: 3} + + Expect(window.Size()).To(Equal(2)) + }) + + It("generates a range of int64s", func() { + numberOfBlocksCreated := history.MakeRange(0, 5) + expected := []int64{0, 1, 2, 3, 4, 5} + + Expect(numberOfBlocksCreated).To(Equal(expected)) + }) + + It("logs window message", func() { + expectedMessage := &bytes.Buffer{} + window := history.ValidationWindow{LowerBound: 5, UpperBound: 7} + history.ParsedWindowTemplate.Execute(expectedMessage, window) + actualMessage := &bytes.Buffer{} + + window.Log(actualMessage) + + Expect(actualMessage).To(Equal(expectedMessage)) + }) +}) diff --git a/test_config/test_config.go b/test_config/test_config.go index b091a0e6..b1a59d1b 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -62,12 +62,13 @@ func setABIPath() { func NewTestDB(node core.Node) *postgres.DB { db, _ := postgres.NewDB(DBConfig, node) - db.MustExec("DELETE FROM watched_contracts") - db.MustExec("DELETE FROM transactions") db.MustExec("DELETE FROM blocks") + db.MustExec("DELETE FROM headers") + db.MustExec("DELETE FROM log_filters") db.MustExec("DELETE FROM logs") db.MustExec("DELETE FROM receipts") - db.MustExec("DELETE FROM log_filters") + db.MustExec("DELETE FROM transactions") + db.MustExec("DELETE FROM watched_contracts") return db }