cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

port over lightSync updates from maker repo

Browse Source
This commit is contained in:
Ian Norden 2018-11-15 12:53:08 -06:00
parent 26aaa8319b
commit 390a60f7f6
40 changed files with 626 additions and 296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
cmd/erc20.go
View File

@ -20,20 +20,14 @@ import (
"log"
"time"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/rpc"
"github.com/spf13/cobra"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/event_triggered/dai"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
"github.com/vulcanize/vulcanizedb/examples/generic"
"github.com/vulcanize/vulcanizedb/libraries/shared"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
"github.com/vulcanize/vulcanizedb/pkg/omni/constants"
"github.com/vulcanize/vulcanizedb/utils"
)
// erc20Cmd represents the erc20 command
@ -62,25 +56,14 @@ Expects an ethereum node to be running and requires a .toml config file:
func watchERC20s() {
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
rawRpcClient, err := rpc.Dial(ipc)
if err != nil {
log.Fatal(err)
}
rpcClient := client.NewRpcClient(rawRpcClient, ipc)
ethClient := ethclient.NewClient(rawRpcClient)
client := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(client, node, transactionConverter)
db, err := postgres.NewDB(databaseConfig, blockChain.Node())
if err != nil {
log.Fatal("Failed to initialize database.")
}
blockChain := getBlockChain()
db := utils.LoadPostgres(databaseConfig, blockChain.Node())
con := generic.DaiConfig
con.Filters = constants.DaiERC20Filters
watcher := shared.Watcher{
DB: *db,
DB: db,
Blockchain: blockChain,
}
@ -89,7 +72,7 @@ func watchERC20s() {
// collect balances and allowances at every block
transformers := append(dai.DaiEventTriggeredTransformerInitializer(), every_block.ERC20EveryBlockTransformerInitializer()...)
err = watcher.AddTransformers(transformers, con)
err := watcher.AddTransformers(transformers, con)
if err != nil {
log.Fatal(err)
}

61
cmd/lightSync.go
View File

@ -14,6 +14,20 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Copyright © 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
@ -21,17 +35,12 @@ import (
"os"
"time"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/rpc"
"github.com/spf13/cobra"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
"github.com/vulcanize/vulcanizedb/pkg/history"
"github.com/vulcanize/vulcanizedb/utils"
)
@ -42,16 +51,12 @@ var lightSyncCmd = &cobra.Command{
Short: "Syncs VulcanizeDB with local ethereum node's block headers",
Long: `Syncs VulcanizeDB with local ethereum node. Populates
Postgres with block headers.
./vulcanizedb lightSync --starting-block-number 0 --config public.toml
Expects ethereum node to be running and requires a .toml config:
[database]
name = "vulcanize_public"
hostname = "localhost"
port = 5432
[client]
ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
`,
@ -66,32 +71,20 @@ func init() {
}
func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
missingBlocksPopulated <- history.PopulateMissingHeaders(blockchain, headerRepository, startingBlockNumber)
populated, err := history.PopulateMissingHeaders(blockchain, headerRepository, startingBlockNumber)
if err != nil {
log.Fatal("Error populating headers: ", err)
}
missingBlocksPopulated <- populated
}
func lightSync() {
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
rawRpcClient, err := rpc.Dial(ipc)
if err != nil {
log.Fatal(err)
}
rpcClient := client.NewRpcClient(rawRpcClient, ipc)
ethClient := ethclient.NewClient(rawRpcClient)
client := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := vRpc.NewRpcTransactionConverter(client)
blockChain := geth.NewBlockChain(client, node, transactionConverter)
lastBlock := blockChain.LastBlock().Int64()
if lastBlock == 0 {
log.Fatal("geth initial: state sync not finished")
}
if startingBlockNumber > lastBlock {
log.Fatal("starting block number > current block number")
}
blockChain := getBlockChain()
validateArgs(blockChain)
db := utils.LoadPostgres(databaseConfig, blockChain.Node())
headerRepository := repositories.NewHeaderRepository(&db)
validator := history.NewHeaderValidator(blockChain, headerRepository, validationWindow)
missingBlocksPopulated := make(chan int)
@ -107,3 +100,13 @@ func lightSync() {
}
}
}
func validateArgs(blockChain *geth.BlockChain) {
lastBlock := blockChain.LastBlock().Int64()
if lastBlock == 0 {
log.Fatal("geth initial: state sync not finished")
}
if startingBlockNumber > lastBlock {
log.Fatal("starting block number > current block number")
}
}

35
cmd/omniWatcher.go
View File

@ -19,23 +19,16 @@ package cmd
import (
"bufio"
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"log"
"os"
"strings"
"time"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/rpc"
"github.com/spf13/cobra"
"github.com/vulcanize/vulcanizedb/libraries/shared"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
"github.com/vulcanize/vulcanizedb/pkg/omni/transformer"
"github.com/vulcanize/vulcanizedb/utils"
)
// omniWatcherCmd represents the omniWatcher command
@ -92,9 +85,9 @@ func omniWatcher() {
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
blockChain, db := setupBCandDB()
t := transformer.NewTransformer(network, blockChain, db)
blockChain := getBlockChain()
db := utils.LoadPostgres(databaseConfig, blockChain.Node())
t := transformer.NewTransformer(network, blockChain, &db)
contractAddresses = append(contractAddresses, contractAddress)
for _, addr := range contractAddresses {
@ -131,23 +124,3 @@ func init() {
omniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block to begin watching- default is first block the contract exists")
omniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "ending-block-number", "d", -1, "Block to end watching- default is most recent block")
}
func setupBCandDB() (core.BlockChain, *postgres.DB) {
rawRpcClient, err := rpc.Dial(ipc)
if err != nil {
log.Fatal(fmt.Sprintf("Failed to initialize rpc client\r\nerr: %v\r\n", err))
}
rpcClient := client.NewRpcClient(rawRpcClient, ipc)
ethClient := ethclient.NewClient(rawRpcClient)
cli := client.NewEthClient(ethClient)
n := node.MakeNode(rpcClient)
transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(cli, n, transactionConverter)
db, err := postgres.NewDB(databaseConfig, blockChain.Node())
if err != nil {
log.Fatal(fmt.Sprintf("Failed to initialize database\r\nerr: %v\r\n", err))
}
return blockChain, db
}

22
cmd/root.go
View File

@ -18,12 +18,20 @@ package cmd
import (
"fmt"
"log"
"os"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/rpc"
"github.com/mitchellh/go-homedir"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
)
var (
@ -109,3 +117,17 @@ func initConfig() {
fmt.Printf("Using config file: %s\n\n", viper.ConfigFileUsed())
}
}
func getBlockChain() *geth.BlockChain {
rawRpcClient, err := rpc.Dial(ipc)
if err != nil {
log.Fatal(err)
}
rpcClient := client.NewRpcClient(rawRpcClient, ipc)
ethClient := ethclient.NewClient(rawRpcClient)
vdbEthClient := client.NewEthClient(ethClient)
vdbNode := node.MakeNode(rpcClient)
transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
return geth.NewBlockChain(vdbEthClient, rpcClient, vdbNode, transactionConverter)
}

18
cmd/sync.go
View File

@ -21,17 +21,11 @@ import (
"os"
"time"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/rpc"
"github.com/spf13/cobra"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
"github.com/vulcanize/vulcanizedb/pkg/history"
"github.com/vulcanize/vulcanizedb/utils"
)
@ -78,16 +72,8 @@ func backFillAllBlocks(blockchain core.BlockChain, blockRepository datastore.Blo
func sync() {
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
rawRpcClient, err := rpc.Dial(ipc)
if err != nil {
log.Fatal(err)
}
rpcClient := client.NewRpcClient(rawRpcClient, ipc)
ethClient := ethclient.NewClient(rawRpcClient)
client := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(client, node, transactionConverter)
blockChain := getBlockChain()
lastBlock := blockChain.LastBlock().Int64()
if lastBlock == 0 {

3
db/migrations/1531758090_create_headers_table.up.sql
View File

@ -2,7 +2,8 @@ CREATE TABLE public.headers (
id SERIAL PRIMARY KEY,
hash VARCHAR(66),
block_number BIGINT,
raw bytea,
raw JSONB,
block_timestamp NUMERIC,
eth_node_id INTEGER,
eth_node_fingerprint VARCHAR(128),
CONSTRAINT eth_nodes_fk FOREIGN KEY (eth_node_id)

1
db/migrations/1532468317_create_checked_headers_table.down.sql Normal file
View File

@ -0,0 +1 @@
DROP TABLE public.checked_headers;

5
db/migrations/1532468317_create_checked_headers_table.up.sql Normal file
View File

@ -0,0 +1,5 @@
CREATE TABLE public.checked_headers (
id SERIAL PRIMARY KEY,
header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
price_feeds_checked BOOLEAN NOT NULL DEFAULT FALSE
);

6
examples/erc20_watcher/every_block/getter_test.go
View File

@ -61,7 +61,7 @@ var _ = Describe("ERC20 Getter", func() {
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
realGetter := every_block.NewGetter(blockChain)
result, err := realGetter.GetTotalSupply(constants.DaiAbiString, constants.DaiContractAddress, blockNumber)
@ -115,7 +115,7 @@ var _ = Describe("ERC20 Getter", func() {
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
realGetter := every_block.NewGetter(blockChain)
testTokenHolderAddress := common.HexToAddress("0x2cccc4b4708b318a6290511aac75d6c3dbe0cf9f")
@ -179,7 +179,7 @@ var _ = Describe("ERC20 Getter", func() {
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
realGetter := every_block.NewGetter(blockChain)
testTokenHolderAddress := common.HexToAddress("0x2cccc4b4708b318a6290511aac75d6c3dbe0cf9f")

14
examples/generic/every_block/getter_test.go
View File

@ -61,7 +61,7 @@ var _ = Describe("every_block Getter", func() {
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
realGetter := every_block.NewGetter(blockChain)
result, err := realGetter.GetStoppedStatus(constants.DaiAbiString, constants.DaiContractAddress, blockNumber)
@ -106,7 +106,7 @@ var _ = Describe("every_block Getter", func() {
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
realGetter := every_block.NewGetter(blockChain)
result, err := realGetter.GetOwner(constants.DaiAbiString, constants.DaiContractAddress, blockNumber)
@ -153,7 +153,7 @@ var _ = Describe("every_block Getter", func() {
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
realGetter := every_block.NewGetter(blockChain)
result, err := realGetter.GetHashName(constants.DaiAbiString, constants.DaiContractAddress, blockNumber)
@ -200,7 +200,7 @@ var _ = Describe("every_block Getter", func() {
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
realGetter := every_block.NewGetter(blockChain)
result, err := realGetter.GetHashSymbol(constants.DaiAbiString, constants.DaiContractAddress, blockNumber)
@ -247,7 +247,7 @@ var _ = Describe("every_block Getter", func() {
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
realGetter := every_block.NewGetter(blockChain)
result, err := realGetter.GetDecimals(constants.DaiAbiString, constants.DaiContractAddress, blockNumber)
@ -295,7 +295,7 @@ var _ = Describe("every_block Getter", func() {
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
realGetter := every_block.NewGetter(blockChain)
result, err := realGetter.GetStringName(constants.TusdAbiString, constants.TusdContractAddress, blockNumber)
@ -342,7 +342,7 @@ var _ = Describe("every_block Getter", func() {
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
realGetter := every_block.NewGetter(blockChain)
result, err := realGetter.GetStringName(constants.TusdAbiString, constants.TusdContractAddress, blockNumber)

4
integration_test/block_rewards_test.go
View File

@ -39,7 +39,7 @@ var _ = Describe("Rewards calculations", func() {
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
block, err := blockChain.GetBlockByNumber(1071819)
Expect(err).ToNot(HaveOccurred())
Expect(block.Reward).To(Equal(5.31355))
@ -53,7 +53,7 @@ var _ = Describe("Rewards calculations", func() {
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
block, err := blockChain.GetBlockByNumber(1071819)
Expect(err).ToNot(HaveOccurred())
Expect(block.UnclesReward).To(Equal(6.875))

6
integration_test/contract_test.go
View File

@ -56,7 +56,7 @@ var _ = Describe("Reading contracts", func() {
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
contract := testing.SampleContract()
logs, err := blockChain.GetLogs(contract, big.NewInt(4703824), nil)
@ -74,7 +74,7 @@ var _ = Describe("Reading contracts", func() {
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
logs, err := blockChain.GetLogs(core.Contract{Hash: "x123"}, big.NewInt(4703824), nil)
@ -93,7 +93,7 @@ var _ = Describe("Reading contracts", func() {
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
contract := testing.SampleContract()
var balance = new(big.Int)

2
integration_test/geth_blockchain_test.go
View File

@ -43,7 +43,7 @@ var _ = Describe("Reading from the Geth blockchain", func() {
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain = geth.NewBlockChain(blockChainClient, node, transactionConverter)
blockChain = geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
})
It("reads two blocks", func(done Done) {

25
pkg/core/header.go
View File

@ -16,8 +16,33 @@
package core
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
)
type Header struct {
Id int64
BlockNumber int64 `db:"block_number"`
Hash string
Raw []byte
Timestamp string `db:"block_timestamp"`
}
type POAHeader struct {
ParentHash common.Hash `json:"parentHash" gencodec:"required"`
UncleHash common.Hash `json:"sha3Uncles" gencodec:"required"`
Coinbase common.Address `json:"miner" gencodec:"required"`
Root common.Hash `json:"stateRoot" gencodec:"required"`
TxHash common.Hash `json:"transactionsRoot" gencodec:"required"`
ReceiptHash common.Hash `json:"receiptsRoot" gencodec:"required"`
Bloom types.Bloom `json:"logsBloom" gencodec:"required"`
Difficulty *hexutil.Big `json:"difficulty" gencodec:"required"`
Number *hexutil.Big `json:"number" gencodec:"required"`
GasLimit hexutil.Uint64 `json:"gasLimit" gencodec:"required"`
GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"`
Time *hexutil.Big `json:"timestamp" gencodec:"required"`
Extra hexutil.Bytes `json:"extraData" gencodec:"required"`
Hash common.Hash `json:"hash"`
}

5
pkg/core/node_info.go
View File

@ -26,6 +26,11 @@ const (
GETH NodeType = iota
PARITY
INFURA
GANACHE
)
const (
KOVAN_NETWORK_ID = 42
)
type Node struct {

1
pkg/datastore/postgres/postgres_test.go
View File

@ -92,6 +92,7 @@ var _ = Describe("Postgres DB", func() {
}
node := core.Node{GenesisBlock: "GENESIS", NetworkID: 1, ID: "x123", ClientName: "geth"}
db := test_config.NewTestDB(node)
test_config.CleanTestDB(db)
blocksRepository := repositories.NewBlockRepository(db)
_, err1 := blocksRepository.CreateOrUpdateBlock(badBlock)

3
pkg/datastore/postgres/repositories/block_repository_test.go
View File

@ -42,6 +42,7 @@ var _ = Describe("Saving blocks", func() {
ClientName: "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
}
db = test_config.NewTestDB(node)
test_config.CleanTestDB(db)
blockRepository = repositories.NewBlockRepository(db)
})
@ -58,6 +59,7 @@ var _ = Describe("Saving blocks", func() {
ClientName: "Geth",
}
dbTwo := test_config.NewTestDB(nodeTwo)
test_config.CleanTestDB(dbTwo)
repositoryTwo := repositories.NewBlockRepository(dbTwo)
_, err := repositoryTwo.GetBlock(123)
@ -184,6 +186,7 @@ var _ = Describe("Saving blocks", func() {
NetworkID: 1,
}
dbTwo := test_config.NewTestDB(nodeTwo)
test_config.CleanTestDB(dbTwo)
repositoryTwo := repositories.NewBlockRepository(dbTwo)
blockRepository.CreateOrUpdateBlock(blockOne)

1
pkg/datastore/postgres/repositories/contract_repository_test.go
View File

@ -41,6 +41,7 @@ var _ = Describe("Creating contracts", func() {
ClientName: "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
}
db = test_config.NewTestDB(node)
test_config.CleanTestDB(db)
contractRepository = repositories.ContractRepository{DB: db}
})

10
pkg/datastore/postgres/repositories/header_repository.go
View File

@ -18,10 +18,14 @@ package repositories
import (
"database/sql"
"errors"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
var ErrValidHeaderExists = errors.New("valid header already exists")
type HeaderRepository struct {
database *postgres.DB
}
@ -41,7 +45,7 @@ func (repository HeaderRepository) CreateOrUpdateHeader(header core.Header) (int
if headerMustBeReplaced(hash, header) {
return repository.replaceHeader(header)
}
return 0, err
return 0, ErrValidHeaderExists
}
func (repository HeaderRepository) GetHeader(blockNumber int64) (core.Header, error) {
@ -81,8 +85,8 @@ func (repository HeaderRepository) getHeaderHash(header core.Header) (string, er
func (repository HeaderRepository) insertHeader(header core.Header) (int64, error) {
var headerId int64
err := repository.database.QueryRowx(
`INSERT INTO public.headers (block_number, hash, raw, eth_node_id, eth_node_fingerprint) VALUES ($1, $2, $3, $4, $5) RETURNING id`,
header.BlockNumber, header.Hash, header.Raw, repository.database.NodeID, repository.database.Node.ID).Scan(&headerId)
`INSERT INTO public.headers (block_number, hash, block_timestamp, raw, eth_node_id, eth_node_fingerprint) VALUES ($1, $2, $3::NUMERIC, $4, $5, $6) RETURNING id`,
header.BlockNumber, header.Hash, header.Timestamp, header.Raw, repository.database.NodeID, repository.database.Node.ID).Scan(&headerId)
return headerId, err
}

122
pkg/datastore/postgres/repositories/header_repository_test.go
View File

@ -18,44 +18,66 @@ package repositories_test
import (
"database/sql"
"encoding/json"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/test_config"
"math/big"
)
var _ = Describe("Block header repository", func() {
Describe("creating or updating a header", func() {
var (
rawHeader []byte
err error
timestamp string
)
BeforeEach(func() {
rawHeader, err = json.Marshal(types.Header{})
Expect(err).NotTo(HaveOccurred())
timestamp = big.NewInt(123456789).String()
})
Describe("creating or updating a header", func() {
It("adds a header", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
test_config.CleanTestDB(db)
repo := repositories.NewHeaderRepository(db)
header := core.Header{
BlockNumber: 100,
Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
Raw: []byte{1, 2, 3, 4, 5},
Raw: rawHeader,
Timestamp: timestamp,
}
_, err := repo.CreateOrUpdateHeader(header)
Expect(err).NotTo(HaveOccurred())
var dbHeader core.Header
err = db.Get(&dbHeader, `SELECT block_number, hash, raw FROM public.headers WHERE block_number = $1`, header.BlockNumber)
err = db.Get(&dbHeader, `SELECT block_number, hash, raw, block_timestamp FROM public.headers WHERE block_number = $1`, header.BlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(dbHeader.BlockNumber).To(Equal(header.BlockNumber))
Expect(dbHeader.Hash).To(Equal(header.Hash))
Expect(dbHeader.Raw).To(Equal(header.Raw))
Expect(dbHeader.Raw).To(MatchJSON(header.Raw))
Expect(dbHeader.Timestamp).To(Equal(header.Timestamp))
})
It("adds node data to header", func() {
node := core.Node{ID: "EthNodeFingerprint"}
db := test_config.NewTestDB(node)
test_config.CleanTestDB(db)
repo := repositories.NewHeaderRepository(db)
header := core.Header{BlockNumber: 100}
header := core.Header{
BlockNumber: 100,
Raw: rawHeader,
Timestamp: timestamp,
}
_, err := repo.CreateOrUpdateHeader(header)
@ -70,21 +92,24 @@ var _ = Describe("Block header repository", func() {
Expect(ethNodeFingerprint).To(Equal(db.Node.ID))
})
It("does not duplicate headers", func() {
It("returns valid header exists error if attempting duplicate headers", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
test_config.CleanTestDB(db)
repo := repositories.NewHeaderRepository(db)
header := core.Header{
BlockNumber: 100,
Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
Raw: []byte{1, 2, 3, 4, 5},
Raw: rawHeader,
Timestamp: timestamp,
}
_, err := repo.CreateOrUpdateHeader(header)
Expect(err).NotTo(HaveOccurred())
_, err = repo.CreateOrUpdateHeader(header)
Expect(err).NotTo(HaveOccurred())
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(repositories.ErrValidHeaderExists))
var dbHeaders []core.Header
err = db.Select(&dbHeaders, `SELECT block_number, hash, raw FROM public.headers WHERE block_number = $1`, header.BlockNumber)
@ -95,18 +120,21 @@ var _ = Describe("Block header repository", func() {
It("replaces header if hash is different", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
test_config.CleanTestDB(db)
repo := repositories.NewHeaderRepository(db)
header := core.Header{
BlockNumber: 100,
Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
Raw: []byte{1, 2, 3, 4, 5},
Raw: rawHeader,
Timestamp: timestamp,
}
_, err := repo.CreateOrUpdateHeader(header)
Expect(err).NotTo(HaveOccurred())
headerTwo := core.Header{
BlockNumber: header.BlockNumber,
Hash: common.BytesToHash([]byte{5, 4, 3, 2, 1}).Hex(),
Raw: []byte{5, 4, 3, 2, 1},
Raw: rawHeader,
Timestamp: timestamp,
}
_, err = repo.CreateOrUpdateHeader(headerTwo)
@ -116,17 +144,19 @@ var _ = Describe("Block header repository", func() {
err = db.Get(&dbHeader, `SELECT block_number, hash, raw FROM headers WHERE block_number = $1`, header.BlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(dbHeader.Hash).To(Equal(headerTwo.Hash))
Expect(dbHeader.Raw).To(Equal(headerTwo.Raw))
Expect(dbHeader.Raw).To(MatchJSON(headerTwo.Raw))
})
It("does not replace header if node fingerprint is different", func() {
node := core.Node{ID: "Fingerprint"}
db := test_config.NewTestDB(node)
test_config.CleanTestDB(db)
repo := repositories.NewHeaderRepository(db)
header := core.Header{
BlockNumber: 100,
Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
Raw: []byte{1, 2, 3, 4, 5},
Raw: rawHeader,
Timestamp: timestamp,
}
_, err := repo.CreateOrUpdateHeader(header)
nodeTwo := core.Node{ID: "FingerprintTwo"}
@ -136,7 +166,8 @@ var _ = Describe("Block header repository", func() {
headerTwo := core.Header{
BlockNumber: header.BlockNumber,
Hash: common.BytesToHash([]byte{5, 4, 3, 2, 1}).Hex(),
Raw: []byte{5, 4, 3, 2, 1},
Raw: rawHeader,
Timestamp: timestamp,
}
_, err = repoTwo.CreateOrUpdateHeader(headerTwo)
@ -155,7 +186,8 @@ var _ = Describe("Block header repository", func() {
header := core.Header{
BlockNumber: 100,
Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
Raw: []byte{1, 2, 3, 4, 5},
Raw: rawHeader,
Timestamp: timestamp,
}
_, err := repo.CreateOrUpdateHeader(header)
nodeTwo := core.Node{ID: "FingerprintTwo"}
@ -165,13 +197,15 @@ var _ = Describe("Block header repository", func() {
headerTwo := core.Header{
BlockNumber: header.BlockNumber,
Hash: common.BytesToHash([]byte{5, 4, 3, 2, 1}).Hex(),
Raw: []byte{5, 4, 3, 2, 1},
Raw: rawHeader,
Timestamp: timestamp,
}
_, err = repoTwo.CreateOrUpdateHeader(headerTwo)
headerThree := core.Header{
BlockNumber: header.BlockNumber,
Hash: common.BytesToHash([]byte{1, 1, 1, 1, 1}).Hex(),
Raw: []byte{1, 1, 1, 1, 1},
Raw: rawHeader,
Timestamp: timestamp,
}
_, err = repoTwo.CreateOrUpdateHeader(headerThree)
@ -183,8 +217,8 @@ var _ = Describe("Block header repository", func() {
Expect(len(dbHeaders)).To(Equal(2))
Expect(dbHeaders[0].Hash).To(Or(Equal(header.Hash), Equal(headerThree.Hash)))
Expect(dbHeaders[1].Hash).To(Or(Equal(header.Hash), Equal(headerThree.Hash)))
Expect(dbHeaders[0].Raw).To(Or(Equal(header.Raw), Equal(headerThree.Raw)))
Expect(dbHeaders[1].Raw).To(Or(Equal(header.Raw), Equal(headerThree.Raw)))
Expect(dbHeaders[0].Raw).To(Or(MatchJSON(header.Raw), MatchJSON(headerThree.Raw)))
Expect(dbHeaders[1].Raw).To(Or(MatchJSON(header.Raw), MatchJSON(headerThree.Raw)))
})
})
@ -192,11 +226,13 @@ var _ = Describe("Block header repository", func() {
It("returns header if it exists", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
test_config.CleanTestDB(db)
repo := repositories.NewHeaderRepository(db)
header := core.Header{
BlockNumber: 100,
Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
Raw: []byte{1, 2, 3, 4, 5},
Raw: rawHeader,
Timestamp: timestamp,
}
_, err := repo.CreateOrUpdateHeader(header)
Expect(err).NotTo(HaveOccurred())
@ -204,17 +240,21 @@ var _ = Describe("Block header repository", func() {
dbHeader, err := repo.GetHeader(header.BlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(dbHeader).To(Equal(header))
Expect(dbHeader.BlockNumber).To(Equal(header.BlockNumber))
Expect(dbHeader.Hash).To(Equal(header.Hash))
Expect(dbHeader.Raw).To(MatchJSON(header.Raw))
})
It("does not return header for a different node fingerprint", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
test_config.CleanTestDB(db)
repo := repositories.NewHeaderRepository(db)
header := core.Header{
BlockNumber: 100,
Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
Raw: []byte{1, 2, 3, 4, 5},
Hash: common.BytesToHash(rawHeader).Hex(),
Raw: rawHeader,
Timestamp: timestamp,
}
_, err := repo.CreateOrUpdateHeader(header)
Expect(err).NotTo(HaveOccurred())
@ -234,10 +274,23 @@ var _ = Describe("Block header repository", func() {
It("returns block numbers for headers not in the database", func() {
node := core.Node{}
db := test_config.NewTestDB(node)
test_config.CleanTestDB(db)
repo := repositories.NewHeaderRepository(db)
repo.CreateOrUpdateHeader(core.Header{BlockNumber: 1})
repo.CreateOrUpdateHeader(core.Header{BlockNumber: 3})
repo.CreateOrUpdateHeader(core.Header{BlockNumber: 5})
repo.CreateOrUpdateHeader(core.Header{
BlockNumber: 1,
Raw: rawHeader,
Timestamp: timestamp,
})
repo.CreateOrUpdateHeader(core.Header{
BlockNumber: 3,
Raw: rawHeader,
Timestamp: timestamp,
})
repo.CreateOrUpdateHeader(core.Header{
BlockNumber: 5,
Raw: rawHeader,
Timestamp: timestamp,
})
missingBlockNumbers := repo.MissingBlockNumbers(1, 5, node.ID)
@ -247,10 +300,23 @@ var _ = Describe("Block header repository", func() {
It("does not count headers created by a different node fingerprint", func() {
node := core.Node{ID: "NodeFingerprint"}
db := test_config.NewTestDB(node)
test_config.CleanTestDB(db)
repo := repositories.NewHeaderRepository(db)
repo.CreateOrUpdateHeader(core.Header{BlockNumber: 1})
repo.CreateOrUpdateHeader(core.Header{BlockNumber: 3})
repo.CreateOrUpdateHeader(core.Header{BlockNumber: 5})
repo.CreateOrUpdateHeader(core.Header{
BlockNumber: 1,
Raw: rawHeader,
Timestamp: timestamp,
})
repo.CreateOrUpdateHeader(core.Header{
BlockNumber: 3,
Raw: rawHeader,
Timestamp: timestamp,
})
repo.CreateOrUpdateHeader(core.Header{
BlockNumber: 5,
Raw: rawHeader,
Timestamp: timestamp,
})
nodeTwo := core.Node{ID: "NodeFingerprintTwo"}
dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo)
Expect(err).NotTo(HaveOccurred())

1
pkg/datastore/postgres/repositories/log_filter_repository_test.go
View File

@ -39,6 +39,7 @@ var _ = Describe("Log Filters Repository", func() {
ClientName: "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
}
db = test_config.NewTestDB(node)
test_config.CleanTestDB(db)
filterRepository = repositories.FilterRepository{DB: db}
})

1
pkg/datastore/postgres/repositories/logs_repository_test.go
View File

@ -44,6 +44,7 @@ var _ = Describe("Logs Repository", func() {
ClientName: "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
}
db = test_config.NewTestDB(node)
test_config.CleanTestDB(db)
blockRepository = repositories.NewBlockRepository(db)
logsRepository = repositories.LogRepository{DB: db}
receiptRepository = repositories.ReceiptRepository{DB: db}

1
pkg/datastore/postgres/repositories/receipts_repository_test.go
View File

@ -40,6 +40,7 @@ var _ = Describe("Receipts Repository", func() {
ClientName: "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
}
db = test_config.NewTestDB(node)
test_config.CleanTestDB(db)
blockRepository = repositories.NewBlockRepository(db)
logRepository = repositories.LogRepository{DB: db}
receiptRepository = repositories.ReceiptRepository{DB: db}

1
pkg/datastore/postgres/repositories/watched_events_repository_test.go
View File

@ -37,6 +37,7 @@ var _ = Describe("Watched Events Repository", func() {
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
blocksRepository = repositories.NewBlockRepository(db)
filterRepository = repositories.FilterRepository{DB: db}
logRepository = repositories.LogRepository{DB: db}

33
pkg/fakes/data.go
View File

@ -16,6 +16,35 @@
package fakes
import "errors"
import (
"encoding/json"
"errors"
"strconv"
var FakeError = errors.New("failed")
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/core"
)
var (
FakeError = errors.New("failed")
FakeHash = common.BytesToHash([]byte{1, 2, 3, 4, 5})
fakeTimestamp = int64(111111111)
)
var rawFakeHeader, _ = json.Marshal(types.Header{})
var FakeHeader = core.Header{
Hash: FakeHash.String(),
Raw: rawFakeHeader,
Timestamp: strconv.FormatInt(fakeTimestamp, 10),
}
func GetFakeHeader(blockNumber int64) core.Header {
return core.Header{
Hash: FakeHash.String(),
BlockNumber: blockNumber,
Raw: rawFakeHeader,
Timestamp: strconv.FormatInt(fakeTimestamp, 10),
}
}

52
pkg/fakes/mock_rpc_client.go
View File

@ -18,29 +18,38 @@ package fakes
import (
"context"
"math/big"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/p2p"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
)
type MockRpcClient struct {
callContextErr error
ipcPath string
nodeType core.NodeType
passedContext context.Context
passedMethod string
passedResult interface{}
returnPOAHeader core.POAHeader
supportedModules map[string]string
}
func NewMockRpcClient() *MockRpcClient {
return &MockRpcClient{}
}
func (client *MockRpcClient) SetNodeType(nodeType core.NodeType) {
client.nodeType = nodeType
}
func (client *MockRpcClient) SetIpcPath(ipcPath string) {
client.ipcPath = ipcPath
}
func (*MockRpcClient) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error {
func (client *MockRpcClient) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error {
client.passedContext = ctx
client.passedResult = result
client.passedMethod = method
switch method {
case "admin_nodeInfo":
if p, ok := result.(*p2p.NodeInfo); ok {
@ -49,9 +58,14 @@ func (*MockRpcClient) CallContext(ctx context.Context, result interface{}, metho
}
case "eth_getBlockByNumber":
if p, ok := result.(*types.Header); ok {
*p = types.Header{}
*p = types.Header{Number: big.NewInt(123)}
}
if p, ok := result.(*core.POAHeader); ok {
*p = client.returnPOAHeader
}
if client.callContextErr != nil {
return client.callContextErr
}
case "parity_versionInfo":
if p, ok := result.(*core.ParityNodeInfo); ok {
*p = core.ParityNodeInfo{
@ -81,9 +95,23 @@ func (client *MockRpcClient) IpcPath() string {
}
func (client *MockRpcClient) SupportedModules() (map[string]string, error) {
result := make(map[string]string)
if client.nodeType == core.GETH {
result["admin"] = "ok"
}
return result, nil
return client.supportedModules, nil
}
func (client *MockRpcClient) SetSupporedModules(supportedModules map[string]string) {
client.supportedModules = supportedModules
}
func (client *MockRpcClient) SetCallContextErr(err error) {
client.callContextErr = err
}
func (client *MockRpcClient) SetReturnPOAHeader(header core.POAHeader) {
client.returnPOAHeader = header
}
func (client *MockRpcClient) AssertCallContextCalledWith(ctx context.Context, result interface{}, method string) {
Expect(client.passedContext).To(Equal(ctx))
Expect(client.passedResult).To(BeAssignableToTypeOf(result))
Expect(client.passedMethod).To(Equal(method))
}

67
pkg/geth/blockchain.go
View File

@ -17,34 +17,41 @@
package geth
import (
"errors"
"math/big"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
"golang.org/x/net/context"
"github.com/vulcanize/vulcanizedb/pkg/core"
vulcCommon "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
)
var ErrEmptyHeader = errors.New("empty header returned over RPC")
type BlockChain struct {
client core.EthClient
blockConverter vulcCommon.BlockConverter
ethClient core.EthClient
headerConverter vulcCommon.HeaderConverter
node core.Node
rpcClient core.RpcClient
}
func NewBlockChain(client core.EthClient, node core.Node, converter vulcCommon.TransactionConverter) *BlockChain {
func NewBlockChain(ethClient core.EthClient, rpcClient core.RpcClient, node core.Node, converter vulcCommon.TransactionConverter) *BlockChain {
return &BlockChain{
client: client,
blockConverter: vulcCommon.NewBlockConverter(converter),
ethClient: ethClient,
headerConverter: vulcCommon.HeaderConverter{},
node: node,
rpcClient: rpcClient,
}
}
func (blockChain *BlockChain) GetBlockByNumber(blockNumber int64) (block core.Block, err error) {
gethBlock, err := blockChain.client.BlockByNumber(context.Background(), big.NewInt(blockNumber))
gethBlock, err := blockChain.ethClient.BlockByNumber(context.Background(), big.NewInt(blockNumber))
if err != nil {
return block, err
}
@ -52,11 +59,46 @@ func (blockChain *BlockChain) GetBlockByNumber(blockNumber int64) (block core.Bl
}
func (blockChain *BlockChain) GetHeaderByNumber(blockNumber int64) (header core.Header, err error) {
gethHeader, err := blockChain.client.HeaderByNumber(context.Background(), big.NewInt(blockNumber))
if blockChain.node.NetworkID == core.KOVAN_NETWORK_ID {
return blockChain.getPOAHeader(blockNumber)
}
return blockChain.getPOWHeader(blockNumber)
}
func (blockChain *BlockChain) getPOWHeader(blockNumber int64) (header core.Header, err error) {
gethHeader, err := blockChain.ethClient.HeaderByNumber(context.Background(), big.NewInt(blockNumber))
if err != nil {
return header, err
}
return blockChain.headerConverter.Convert(gethHeader)
return blockChain.headerConverter.Convert(gethHeader, gethHeader.Hash().String())
}
func (blockChain *BlockChain) getPOAHeader(blockNumber int64) (header core.Header, err error) {
var POAHeader core.POAHeader
blockNumberArg := hexutil.EncodeBig(big.NewInt(blockNumber))
includeTransactions := false
err = blockChain.rpcClient.CallContext(context.Background(), &POAHeader, "eth_getBlockByNumber", blockNumberArg, includeTransactions)
if err != nil {
return header, err
}
if POAHeader.Number == nil {
return header, ErrEmptyHeader
}
return blockChain.headerConverter.Convert(&types.Header{
ParentHash: POAHeader.ParentHash,
UncleHash: POAHeader.UncleHash,
Coinbase: POAHeader.Coinbase,
Root: POAHeader.Root,
TxHash: POAHeader.TxHash,
ReceiptHash: POAHeader.ReceiptHash,
Bloom: POAHeader.Bloom,
Difficulty: POAHeader.Difficulty.ToInt(),
Number: POAHeader.Number.ToInt(),
GasLimit: uint64(POAHeader.GasLimit),
GasUsed: uint64(POAHeader.GasUsed),
Time: POAHeader.Time.ToInt(),
Extra: POAHeader.Extra,
}, POAHeader.Hash.String())
}
func (blockChain *BlockChain) GetLogs(contract core.Contract, startingBlockNumber, endingBlockNumber *big.Int) ([]core.Log, error) {
@ -68,8 +110,9 @@ func (blockChain *BlockChain) GetLogs(contract core.Contract, startingBlockNumbe
FromBlock: startingBlockNumber,
ToBlock: endingBlockNumber,
Addresses: []common.Address{contractAddress},
Topics: nil,
}
gethLogs, err := blockChain.client.FilterLogs(context.Background(), fc)
gethLogs, err := blockChain.GetEthLogsWithCustomQuery(fc)
if err != nil {
return []core.Log{}, err
}
@ -77,8 +120,16 @@ func (blockChain *BlockChain) GetLogs(contract core.Contract, startingBlockNumbe
return logs, nil
}
func (blockChain *BlockChain) GetEthLogsWithCustomQuery(query ethereum.FilterQuery) ([]types.Log, error) {
gethLogs, err := blockChain.ethClient.FilterLogs(context.Background(), query)
if err != nil {
return []types.Log{}, err
}
return gethLogs, nil
}
func (blockChain *BlockChain) LastBlock() *big.Int {
block, _ := blockChain.client.HeaderByNumber(context.Background(), nil)
block, _ := blockChain.ethClient.HeaderByNumber(context.Background(), nil)
return block.Number
}

126
pkg/geth/blockchain_test.go
View File

@ -22,6 +22,7 @@ import (
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -33,12 +34,21 @@ import (
)
var _ = Describe("Geth blockchain", func() {
var mockClient *fakes.MockEthClient
var mockRpcClient *fakes.MockRpcClient
var node vulcCore.Node
var blockChain *geth.BlockChain
BeforeEach(func() {
mockClient = fakes.NewMockEthClient()
mockRpcClient = fakes.NewMockRpcClient()
node = vulcCore.Node{}
blockChain = geth.NewBlockChain(mockClient, mockRpcClient, node, cold_db.NewColdDbTransactionConverter())
})
Describe("getting a block", func() {
It("fetches block from client", func() {
mockClient := fakes.NewMockEthClient()
It("fetches block from ethClient", func() {
mockClient.SetBlockByNumberReturnBlock(types.NewBlockWithHeader(&types.Header{}))
node := vulcCore.Node{}
blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
blockNumber := int64(100)
_, err := blockChain.GetBlockByNumber(blockNumber)
@ -47,11 +57,8 @@ var _ = Describe("Geth blockchain", func() {
mockClient.AssertBlockByNumberCalledWith(context.Background(), big.NewInt(blockNumber))
})
It("returns err if client returns err", func() {
mockClient := fakes.NewMockEthClient()
It("returns err if ethClient returns err", func() {
mockClient.SetBlockByNumberErr(fakes.FakeError)
node := vulcCore.Node{}
blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
_, err := blockChain.GetBlockByNumber(100)
@ -61,12 +68,10 @@ var _ = Describe("Geth blockchain", func() {
})
Describe("getting a header", func() {
It("fetches header from client", func() {
mockClient := fakes.NewMockEthClient()
Describe("default/mainnet", func() {
It("fetches header from ethClient", func() {
blockNumber := int64(100)
mockClient.SetHeaderByNumberReturnHeader(&types.Header{Number: big.NewInt(blockNumber)})
node := vulcCore.Node{}
blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
_, err := blockChain.GetHeaderByNumber(blockNumber)
@ -74,11 +79,8 @@ var _ = Describe("Geth blockchain", func() {
mockClient.AssertHeaderByNumberCalledWith(context.Background(), big.NewInt(blockNumber))
})
It("returns err if client returns err", func() {
mockClient := fakes.NewMockEthClient()
It("returns err if ethClient returns err", func() {
mockClient.SetHeaderByNumberErr(fakes.FakeError)
node := vulcCore.Node{}
blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
_, err := blockChain.GetHeaderByNumber(100)
@ -87,12 +89,45 @@ var _ = Describe("Geth blockchain", func() {
})
})
Describe("getting logs", func() {
It("fetches logs from client", func() {
mockClient := fakes.NewMockEthClient()
Describe("POA/Kovan", func() {
It("fetches header from rpcClient", func() {
node.NetworkID = vulcCore.KOVAN_NETWORK_ID
blockNumber := hexutil.Big(*big.NewInt(123))
mockRpcClient.SetReturnPOAHeader(vulcCore.POAHeader{Number: &blockNumber})
blockChain = geth.NewBlockChain(mockClient, mockRpcClient, node, cold_db.NewColdDbTransactionConverter())
_, err := blockChain.GetHeaderByNumber(100)
Expect(err).NotTo(HaveOccurred())
mockRpcClient.AssertCallContextCalledWith(context.Background(), &vulcCore.POAHeader{}, "eth_getBlockByNumber")
})
It("returns err if rpcClient returns err", func() {
node.NetworkID = vulcCore.KOVAN_NETWORK_ID
mockRpcClient.SetCallContextErr(fakes.FakeError)
blockChain = geth.NewBlockChain(mockClient, mockRpcClient, node, cold_db.NewColdDbTransactionConverter())
_, err := blockChain.GetHeaderByNumber(100)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("returns error if returned header is empty", func() {
node.NetworkID = vulcCore.KOVAN_NETWORK_ID
blockChain = geth.NewBlockChain(mockClient, mockRpcClient, node, cold_db.NewColdDbTransactionConverter())
_, err := blockChain.GetHeaderByNumber(100)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(geth.ErrEmptyHeader))
})
})
})
Describe("getting logs with default FilterQuery", func() {
It("fetches logs from ethClient", func() {
mockClient.SetFilterLogsReturnLogs([]types.Log{{}})
node := vulcCore.Node{}
blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
contract := vulcCore.Contract{Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex()}
startingBlockNumber := big.NewInt(1)
endingBlockNumber := big.NewInt(2)
@ -108,11 +143,8 @@ var _ = Describe("Geth blockchain", func() {
mockClient.AssertFilterLogsCalledWith(context.Background(), expectedQuery)
})
It("returns err if client returns err", func() {
mockClient := fakes.NewMockEthClient()
It("returns err if ethClient returns err", func() {
mockClient.SetFilterLogsErr(fakes.FakeError)
node := vulcCore.Node{}
blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
contract := vulcCore.Contract{Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex()}
startingBlockNumber := big.NewInt(1)
endingBlockNumber := big.NewInt(2)
@ -124,13 +156,49 @@ var _ = Describe("Geth blockchain", func() {
})
})
Describe("getting logs with a custom FilterQuery", func() {
It("fetches logs from ethClient", func() {
mockClient.SetFilterLogsReturnLogs([]types.Log{{}})
address := common.HexToAddress("0x")
startingBlockNumber := big.NewInt(1)
endingBlockNumber := big.NewInt(2)
topic := common.HexToHash("0x")
query := ethereum.FilterQuery{
FromBlock: startingBlockNumber,
ToBlock: endingBlockNumber,
Addresses: []common.Address{address},
Topics: [][]common.Hash{{topic}},
}
_, err := blockChain.GetEthLogsWithCustomQuery(query)
Expect(err).NotTo(HaveOccurred())
mockClient.AssertFilterLogsCalledWith(context.Background(), query)
})
It("returns err if ethClient returns err", func() {
mockClient.SetFilterLogsErr(fakes.FakeError)
contract := vulcCore.Contract{Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex()}
startingBlockNumber := big.NewInt(1)
endingBlockNumber := big.NewInt(2)
query := ethereum.FilterQuery{
FromBlock: startingBlockNumber,
ToBlock: endingBlockNumber,
Addresses: []common.Address{common.HexToAddress(contract.Hash)},
Topics: nil,
}
_, err := blockChain.GetEthLogsWithCustomQuery(query)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
})
Describe("getting the most recent block number", func() {
It("fetches latest header from client", func() {
mockClient := fakes.NewMockEthClient()
It("fetches latest header from ethClient", func() {
blockNumber := int64(100)
mockClient.SetHeaderByNumberReturnHeader(&types.Header{Number: big.NewInt(blockNumber)})
node := vulcCore.Node{}
blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
result := blockChain.LastBlock()

5
pkg/geth/contract.go
View File

@ -17,9 +17,8 @@
package geth
import (
"errors"
"context"
"errors"
"math/big"
"github.com/ethereum/go-ethereum"
@ -54,5 +53,5 @@ func (blockChain *BlockChain) FetchContractData(abiJSON string, address string,
func (blockChain *BlockChain) callContract(contractHash string, input []byte, blockNumber *big.Int) ([]byte, error) {
to := common.HexToAddress(contractHash)
msg := ethereum.CallMsg{To: &to, Data: input}
return blockChain.client.CallContract(context.Background(), msg, blockNumber)
return blockChain.ethClient.CallContract(context.Background(), msg, blockNumber)
}

15
pkg/geth/converters/common/header_converter.go
View File

@ -17,24 +17,25 @@
package common
import (
"bytes"
"encoding/json"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
"github.com/vulcanize/vulcanizedb/pkg/core"
)
type HeaderConverter struct{}
func (converter HeaderConverter) Convert(gethHeader *types.Header) (core.Header, error) {
writer := new(bytes.Buffer)
err := rlp.Encode(writer, &gethHeader)
func (converter HeaderConverter) Convert(gethHeader *types.Header, blockHash string) (core.Header, error) {
rawHeader, err := json.Marshal(gethHeader)
if err != nil {
panic(err)
}
coreHeader := core.Header{
Hash: gethHeader.Hash().Hex(),
Hash: blockHash,
BlockNumber: gethHeader.Number.Int64(),
Raw: writer.Bytes(),
Raw: rawHeader,
Timestamp: gethHeader.Time.String(),
}
return coreHeader, nil
}

29
pkg/geth/converters/common/header_converter_test.go
View File

@ -17,14 +17,16 @@
package common_test
import (
"bytes"
"encoding/json"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
common2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
"math/big"
)
var _ = Describe("Block header converter", func() {
@ -35,29 +37,30 @@ var _ = Describe("Block header converter", func() {
ParentHash: common.HexToHash("0xParent"),
ReceiptHash: common.HexToHash("0xReceipt"),
Root: common.HexToHash("0xRoot"),
Time: big.NewInt(3),
Time: big.NewInt(123456789),
TxHash: common.HexToHash("0xTransaction"),
UncleHash: common.HexToHash("0xUncle"),
}
converter := common2.HeaderConverter{}
hash := fakes.FakeHash.String()
coreHeader, err := converter.Convert(gethHeader)
coreHeader, err := converter.Convert(gethHeader, hash)
Expect(err).NotTo(HaveOccurred())
Expect(coreHeader.BlockNumber).To(Equal(gethHeader.Number.Int64()))
Expect(coreHeader.Hash).To(Equal(gethHeader.Hash().Hex()))
Expect(coreHeader.Hash).To(Equal(hash))
Expect(coreHeader.Timestamp).To(Equal(gethHeader.Time.String()))
})
It("includes raw bytes for header", func() {
headerRLP := []byte{249, 2, 23, 160, 180, 251, 173, 248, 234, 69, 43, 19, 151, 24, 226, 112, 13, 193, 19, 92, 252, 129, 20, 80, 49, 200, 75, 122, 178, 124, 215, 16, 57, 79, 123, 56, 160, 29, 204, 77, 232, 222, 199, 93, 122, 171, 133, 181, 103, 182, 204, 212, 26, 211, 18, 69, 27, 148, 138, 116, 19, 240, 161, 66, 253, 64, 212, 147, 71, 148, 42, 101, 172, 164, 213, 252, 91, 92, 133, 144, 144, 166, 195, 77, 22, 65, 53, 57, 130, 38, 160, 14, 6, 111, 60, 34, 151, 165, 203, 48, 5, 147, 5, 38, 23, 209, 188, 165, 148, 111, 12, 170, 6, 53, 253, 177, 184, 90, 199, 229, 35, 111, 52, 160, 101, 186, 136, 127, 203, 8, 38, 246, 22, 208, 31, 115, 108, 29, 45, 103, 123, 202, 189, 226, 247, 252, 37, 170, 145, 207, 188, 11, 59, 173, 92, 179, 160, 32, 227, 83, 69, 64, 202, 241, 99, 120, 230, 232, 106, 43, 241, 35, 109, 159, 135, 109, 50, 24, 251, 192, 57, 88, 230, 219, 28, 99, 75, 35, 51, 185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 134, 11, 105, 222, 129, 162, 43, 131, 15, 66, 64, 131, 47, 239, 216, 130, 196, 68, 132, 86, 191, 180, 21, 152, 215, 131, 1, 3, 3, 132, 71, 101, 116, 104, 135, 103, 111, 49, 46, 53, 46, 49, 133, 108, 105, 110, 117, 120, 160, 146, 196, 18, 154, 10, 226, 54, 27, 69, 42, 158, 222, 236, 229, 92, 18, 236, 238, 171, 134, 99, 22, 25, 94, 61, 135, 252, 27, 0, 91, 102, 69, 136, 205, 76, 85, 185, 65, 207, 144, 21}
var gethHeader types.Header
err := rlp.Decode(bytes.NewReader(headerRLP), &gethHeader)
Expect(err).NotTo(HaveOccurred())
It("includes raw bytes for header as JSON", func() {
gethHeader := types.Header{Number: big.NewInt(123)}
converter := common2.HeaderConverter{}
coreHeader, err := converter.Convert(&gethHeader)
coreHeader, err := converter.Convert(&gethHeader, fakes.FakeHash.String())
Expect(err).NotTo(HaveOccurred())
Expect(coreHeader.Raw).To(Equal(headerRLP))
expectedJSON, err := json.Marshal(gethHeader)
Expect(err).NotTo(HaveOccurred())
Expect(coreHeader.Raw).To(Equal(expectedJSON))
})
})

13
pkg/geth/node/node.go
View File

@ -53,6 +53,10 @@ type InfuraClient struct {
PropertiesReader
}
type GanacheClient struct {
PropertiesReader
}
func MakeNode(rpcClient core.RpcClient) core.Node {
pr := makePropertiesReader(rpcClient)
id, name := pr.NodeInfo()
@ -72,6 +76,8 @@ func makePropertiesReader(client core.RpcClient) IPropertiesReader {
return ParityClient{PropertiesReader: PropertiesReader{client: client}}
case core.INFURA:
return InfuraClient{PropertiesReader: PropertiesReader{client: client}}
case core.GANACHE:
return GanacheClient{PropertiesReader: PropertiesReader{client: client}}
default:
return PropertiesReader{client: client}
}
@ -81,6 +87,9 @@ func getNodeType(client core.RpcClient) core.NodeType {
if strings.Contains(client.IpcPath(), "infura") {
return core.INFURA
}
if strings.Contains(client.IpcPath(), "127.0.0.1") || strings.Contains(client.IpcPath(), "localhost") {
return core.GANACHE
}
modules, _ := client.SupportedModules()
if _, ok := modules["admin"]; ok {
return core.GETH
@ -122,6 +131,10 @@ func (client InfuraClient) NodeInfo() (string, string) {
return "infura", "infura"
}
func (client GanacheClient) NodeInfo() (string, string) {
return "ganache", "ganache"
}
func (client ParityClient) parityNodeInfo() string {
var nodeInfo core.ParityNodeInfo
client.client.CallContext(context.Background(), &nodeInfo, "parity_versionInfo")

41
pkg/geth/node/node_test.go
View File

@ -21,6 +21,7 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
@ -28,8 +29,8 @@ import (
var EmpytHeaderHash = "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"
var _ = Describe("Parity Node Info", func() {
var _ = Describe("Node Info", func() {
Describe("Parity Node Info", func() {
It("verifies parity_versionInfo can be unmarshalled into ParityNodeInfo", func() {
var parityNodeInfo core.ParityNodeInfo
nodeInfoJSON := []byte(
@ -63,6 +64,15 @@ var _ = Describe("Parity Node Info", func() {
Expect(parityNodeInfo.String()).To(Equal("Parity/v1.6.0/"))
})
It("returns parity ID and client name for parity node", func() {
client := fakes.NewMockRpcClient()
n := node.MakeNode(client)
Expect(n.ID).To(Equal("ParityNode"))
Expect(n.ClientName).To(Equal("Parity/v1.2.3/"))
})
})
It("returns the genesis block for any client", func() {
client := fakes.NewMockRpcClient()
n := node.MakeNode(client)
@ -75,17 +85,12 @@ var _ = Describe("Parity Node Info", func() {
Expect(n.NetworkID).To(Equal(float64(1234)))
})
It("returns parity ID and client name for parity node", func() {
client := fakes.NewMockRpcClient()
client.SetNodeType(core.PARITY)
n := node.MakeNode(client)
Expect(n.ID).To(Equal("ParityNode"))
Expect(n.ClientName).To(Equal("Parity/v1.2.3/"))
})
It("returns geth ID and client name for geth node", func() {
client := fakes.NewMockRpcClient()
client.SetNodeType(core.GETH)
supportedModules := make(map[string]string)
supportedModules["admin"] = "ok"
client.SetSupporedModules(supportedModules)
n := node.MakeNode(client)
Expect(n.ID).To(Equal("enode://GethNode@172.17.0.1:30303"))
Expect(n.ClientName).To(Equal("Geth/v1.7"))
@ -93,10 +98,22 @@ var _ = Describe("Parity Node Info", func() {
It("returns infura ID and client name for infura node", func() {
client := fakes.NewMockRpcClient()
client.SetNodeType(core.INFURA)
client.SetIpcPath("infura/path")
n := node.MakeNode(client)
Expect(n.ID).To(Equal("infura"))
Expect(n.ClientName).To(Equal("infura"))
})
It("returns local id and client name for Local node", func() {
client := fakes.NewMockRpcClient()
client.SetIpcPath("127.0.0.1")
n := node.MakeNode(client)
Expect(n.ID).To(Equal("ganache"))
Expect(n.ClientName).To(Equal("ganache"))
client.SetIpcPath("localhost")
n = node.MakeNode(client)
Expect(n.ID).To(Equal("ganache"))
Expect(n.ClientName).To(Equal("ganache"))
})
})

31
pkg/history/populate_headers.go
View File

@ -17,29 +17,38 @@
package history
import (
"log"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"log"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
)
func PopulateMissingHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, startingBlockNumber int64) int {
func PopulateMissingHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, startingBlockNumber int64) (int, error) {
lastBlock := blockchain.LastBlock().Int64()
blockRange := headerRepository.MissingBlockNumbers(startingBlockNumber, lastBlock, blockchain.Node().ID)
log.SetPrefix("")
log.Printf("Backfilling %d blocks\n\n", len(blockRange))
RetrieveAndUpdateHeaders(blockchain, headerRepository, blockRange)
return len(blockRange)
_, err := RetrieveAndUpdateHeaders(blockchain, headerRepository, blockRange)
if err != nil {
return 0, err
}
return len(blockRange), nil
}
func RetrieveAndUpdateHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, blockNumbers []int64) int {
func RetrieveAndUpdateHeaders(chain core.BlockChain, headerRepository datastore.HeaderRepository, blockNumbers []int64) (int, error) {
for _, blockNumber := range blockNumbers {
header, err := blockchain.GetHeaderByNumber(blockNumber)
header, err := chain.GetHeaderByNumber(blockNumber)
if err != nil {
log.Printf("failed to retrieve block number: %d\n", blockNumber)
return 0
return 0, err
}
// TODO: handle possible error here
headerRepository.CreateOrUpdateHeader(header)
_, err = headerRepository.CreateOrUpdateHeader(header)
if err != nil {
if err == repositories.ErrValidHeaderExists {
continue
}
return len(blockNumbers)
return 0, err
}
}
return len(blockNumbers), nil
}

9
pkg/history/populate_headers_test.go
View File

@ -34,26 +34,25 @@ var _ = Describe("Populating headers", func() {
headerRepository = fakes.NewMockHeaderRepository()
})
Describe("When 1 missing header", func() {
It("returns number of headers added", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(2))
headerRepository.SetMissingBlockNumbers([]int64{2})
headersAdded := history.PopulateMissingHeaders(blockChain, headerRepository, 1)
headersAdded, err := history.PopulateMissingHeaders(blockChain, headerRepository, 1)
Expect(err).NotTo(HaveOccurred())
Expect(headersAdded).To(Equal(1))
})
})
It("adds missing headers to the db", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(2))
headerRepository.SetMissingBlockNumbers([]int64{2})
history.PopulateMissingHeaders(blockChain, headerRepository, 1)
_, err := history.PopulateMissingHeaders(blockChain, headerRepository, 1)
Expect(err).NotTo(HaveOccurred())
headerRepository.AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(1, []int64{2})
})
})

12
pkg/omni/converter/converter.go
View File

@ -74,26 +74,26 @@ func (c *converter) Convert(watchedEvent core.WatchedEvent, event *types.Event)
strValues := make(map[string]string, len(values))
for eventName, input := range values {
for fieldName, input := range values {
// Postgres cannot handle custom types, resolve to strings
switch input.(type) {
case *big.Int:
var b *big.Int
b = input.(*big.Int)
strValues[eventName] = b.String()
strValues[fieldName] = b.String()
case common.Address:
var a common.Address
a = input.(common.Address)
strValues[eventName] = a.String()
strValues[fieldName] = a.String()
c.contractInfo.AddTokenHolderAddress(a.String()) // cache address in a list of contract's token holder addresses
case common.Hash:
var h common.Hash
h = input.(common.Hash)
strValues[eventName] = h.String()
strValues[fieldName] = h.String()
case string:
strValues[eventName] = input.(string)
strValues[fieldName] = input.(string)
case bool:
strValues[eventName] = strconv.FormatBool(input.(bool))
strValues[fieldName] = strconv.FormatBool(input.(bool))
default:
return errors.New("error: unhandled abi type")
}

4
pkg/omni/helpers/test_helpers/database.go
View File

@ -75,7 +75,7 @@ func SetupBC() core.BlockChain {
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
return blockChain
}
@ -89,7 +89,7 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) {
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
db, err := postgres.NewDB(config.Database{
Hostname: "localhost",

1
pkg/omni/repository/event_repository.go
View File

@ -47,6 +47,7 @@ func NewEventDataStore(db *postgres.DB) *eventDatastore {
// Creates a schema for the contract
// Creates tables for the watched contract events
// Persists converted event log data into these custom tables
// Edit this method to accept a single event
func (d *eventDatastore) PersistContractEvents(con *contract.Contract) error {
_, err := d.CreateContractSchema(con.Address)
if err != nil {

2
pkg/omni/transformer/transformer.go
View File

@ -209,6 +209,8 @@ func (tr transformer) Execute() error {
}
// After converting all logs for events of interest, persist all of the data
// Change this so that event logs are persisted immediately after being created
// So as to prevent a huge growth of data in the contract memory
err := tr.PersistContractEvents(con)
if err != nil {
return err

29
test_config/test_config.go
View File

@ -20,15 +20,18 @@ import (
"log"
"os"
. "github.com/onsi/gomega"
"github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
)
var TestConfig *viper.Viper
var DBConfig config.Database
var TestClient config.Client
var Infura *viper.Viper
var InfuraClient config.Client
var ABIFilePath string
@ -44,6 +47,7 @@ func setTestConfig() {
TestConfig.SetConfigName("private")
TestConfig.AddConfigPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/environments/")
err := TestConfig.ReadInConfig()
ipc := TestConfig.GetString("client.ipcPath")
if err != nil {
log.Fatal(err)
}
@ -55,6 +59,9 @@ func setTestConfig() {
Name: name,
Port: port,
}
TestClient = config.Client{
IPCPath: ipc,
}
}
func setInfuraConfig() {
@ -78,14 +85,34 @@ func setABIPath() {
func NewTestDB(node core.Node) *postgres.DB {
db, _ := postgres.NewDB(DBConfig, node)
return db
}
func CleanTestDB(db *postgres.DB) {
db.MustExec("DELETE FROM blocks")
db.MustExec("DELETE FROM headers")
db.MustExec("DELETE FROM checked_headers")
db.MustExec("DELETE FROM log_filters")
db.MustExec("DELETE FROM logs")
db.MustExec("DELETE FROM receipts")
db.MustExec("DELETE FROM transactions")
db.MustExec("DELETE FROM watched_contracts")
return db
}
func NewTestNode() core.Node {
return core.Node{
GenesisBlock: "GENESIS",
NetworkID: 1,
ID: "b6f90c0fdd8ec9607aed8ee45c69322e47b7063f0bfb7a29c8ecafab24d0a22d24dd2329b5ee6ed4125a03cb14e57fd584e67f9e53e6c631055cbbd82f080845",
ClientName: "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
}
}
func NewTestBlock(blockNumber int64, repository repositories.BlockRepository) (blockId int64) {
blockId, err := repository.CreateOrUpdateBlock(core.Block{Number: blockNumber})
Expect(err).NotTo(HaveOccurred())
return blockId
}
func NewTestDBWithoutDeletingRecords(node core.Node) *postgres.DB {