deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

Sync only missing blocks on cold import

Browse Source 
= Add eth_node_fingerprint to block that can be imitated by both hot and cold imports
- Only sync missing blocks (blocks that are missing or don't share a fingerprint) on cold import
- Set block is_final status after import
This commit is contained in:
Rob Mulholand
2018-05-07 16:38:04 -05:00
parent 5a5e08bd13
commit d5c2ab33fc
31 changed files with 616 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/datastore/inmemory/block_repository.go
+1 -1
View File
@@ -26,7 +26,7 @@ func (blockRepository *BlockRepository) GetBlock(blockNumber int64) (core.Block,
return core.Block{}, datastore.ErrBlockDoesNotExist(blockNumber)
}
func (blockRepository *BlockRepository) MissingBlockNumbers(startingBlockNumber int64, endingBlockNumber int64) []int64 {
func (blockRepository *BlockRepository) MissingBlockNumbers(startingBlockNumber int64, endingBlockNumber int64, nodeId string) []int64 {
missingNumbers := []int64{}
for blockNumber := int64(startingBlockNumber); blockNumber <= endingBlockNumber; blockNumber++ {
if _, ok := blockRepository.blocks[blockNumber]; !ok {
pkg/datastore/postgres/postgres_test.go
+2 -2
View File
@@ -76,7 +76,7 @@ var _ = Describe("Postgres DB", func() {
}
node := core.Node{GenesisBlock: "GENESIS", NetworkID: 1, ID: "x123", ClientName: "geth"}
db := test_config.NewTestDB(node)
blocksRepository := repositories.BlockRepository{DB: db}
blocksRepository := repositories.NewBlockRepository(db)
_, err1 := blocksRepository.CreateOrUpdateBlock(badBlock)
@@ -131,7 +131,7 @@ var _ = Describe("Postgres DB", func() {
}
node := core.Node{GenesisBlock: "GENESIS", NetworkID: 1, ID: "x123", ClientName: "geth"}
db, _ := postgres.NewDB(test_config.DBConfig, node)
blockRepository := repositories.BlockRepository{DB: db}
blockRepository := repositories.NewBlockRepository(db)
_, err1 := blockRepository.CreateOrUpdateBlock(block)
pkg/datastore/postgres/repositories/block_repository.go
+22 -18
View File
@@ -20,12 +20,16 @@ const (
var ErrBlockExists = errors.New("Won't add block that already exists.")
type BlockRepository struct {
*postgres.DB
database *postgres.DB
}
func NewBlockRepository(database *postgres.DB) *BlockRepository {
return &BlockRepository{database: database}
}
func (blockRepository BlockRepository) SetBlocksStatus(chainHead int64) {
cutoff := chainHead - blocksFromHeadBeforeFinal
blockRepository.DB.Exec(`
blockRepository.database.Exec(`
UPDATE blocks SET is_final = TRUE
WHERE is_final = FALSE AND number < $1`,
cutoff)
@@ -48,22 +52,22 @@ func (blockRepository BlockRepository) CreateOrUpdateBlock(block core.Block) (in
return blockId, ErrBlockExists
}
func (blockRepository BlockRepository) MissingBlockNumbers(startingBlockNumber int64, highestBlockNumber int64) []int64 {
func (blockRepository BlockRepository) MissingBlockNumbers(startingBlockNumber int64, highestBlockNumber int64, nodeId string) []int64 {
numbers := make([]int64, 0)
blockRepository.DB.Select(&numbers,
blockRepository.database.Select(&numbers,
`SELECT all_block_numbers
FROM (
SELECT generate_series($1::INT, $2::INT) AS all_block_numbers) series
LEFT JOIN blocks
ON number = all_block_numbers
WHERE number ISNULL`,
WHERE number ISNULL OR eth_node_fingerprint != $3`,
startingBlockNumber,
highestBlockNumber)
highestBlockNumber, nodeId)
return numbers
}
func (blockRepository BlockRepository) GetBlock(blockNumber int64) (core.Block, error) {
blockRows := blockRepository.DB.QueryRowx(
blockRows := blockRepository.database.QueryRowx(
`SELECT id,
number,
gaslimit,
@@ -81,7 +85,7 @@ func (blockRepository BlockRepository) GetBlock(blockNumber int64) (core.Block,
reward,
uncles_reward
FROM blocks
WHERE eth_node_id = $1 AND number = $2`, blockRepository.NodeID, blockNumber)
WHERE eth_node_id = $1 AND number = $2`, blockRepository.database.NodeID, blockNumber)
savedBlock, err := blockRepository.loadBlock(blockRows)
if err != nil {
switch err {
@@ -96,17 +100,17 @@ func (blockRepository BlockRepository) GetBlock(blockNumber int64) (core.Block,
func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, error) {
var blockId int64
tx, _ := blockRepository.DB.BeginTx(context.Background(), nil)
tx, _ := blockRepository.database.BeginTx(context.Background(), nil)
err := tx.QueryRow(
`INSERT INTO blocks
(eth_node_id, number, gaslimit, gasused, time, difficulty, hash, nonce, parenthash, size, uncle_hash, is_final, miner, extra_data, reward, uncles_reward)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
(eth_node_id, number, gaslimit, gasused, time, difficulty, hash, nonce, parenthash, size, uncle_hash, is_final, miner, extra_data, reward, uncles_reward, eth_node_fingerprint)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
RETURNING id `,
blockRepository.NodeID, block.Number, block.GasLimit, block.GasUsed, block.Time, block.Difficulty, block.Hash, block.Nonce, block.ParentHash, block.Size, block.UncleHash, block.IsFinal, block.Miner, block.ExtraData, block.Reward, block.UnclesReward).
blockRepository.database.NodeID, block.Number, block.GasLimit, block.GasUsed, block.Time, block.Difficulty, block.Hash, block.Nonce, block.ParentHash, block.Size, block.UncleHash, block.IsFinal, block.Miner, block.ExtraData, block.Reward, block.UnclesReward, blockRepository.database.Node.ID).
Scan(&blockId)
if err != nil {
tx.Rollback()
return 0, postgres.ErrDBInsertFailed
return 0, err
}
if len(block.Transactions) > 0 {
err = blockRepository.createTransactions(tx, blockId, block.Transactions)
@@ -189,11 +193,11 @@ func (blockRepository BlockRepository) createReceipt(tx *sql.Tx, blockId int64,
func (blockRepository BlockRepository) getBlockHash(block core.Block) (string, bool) {
var retrievedBlockHash string
blockRepository.DB.Get(&retrievedBlockHash,
blockRepository.database.Get(&retrievedBlockHash,
`SELECT hash
FROM blocks
WHERE number = $1 AND eth_node_id = $2`,
block.Number, blockRepository.NodeID)
block.Number, blockRepository.database.NodeID)
return retrievedBlockHash, blockExists(retrievedBlockHash)
}
@@ -217,11 +221,11 @@ func blockExists(retrievedBlockHash string) bool {
}
func (blockRepository BlockRepository) removeBlock(blockNumber int64) error {
_, err := blockRepository.DB.Exec(
_, err := blockRepository.database.Exec(
`DELETE FROM
blocks
WHERE number=$1 AND eth_node_id=$2`,
blockNumber, blockRepository.NodeID)
blockNumber, blockRepository.database.NodeID)
if err != nil {
return postgres.ErrDBDeleteFailed
}
@@ -238,7 +242,7 @@ func (blockRepository BlockRepository) loadBlock(blockRows *sqlx.Row) (core.Bloc
if err != nil {
return core.Block{}, err
}
transactionRows, err := blockRepository.DB.Queryx(`
transactionRows, err := blockRepository.database.Queryx(`
SELECT hash,
nonce,
tx_to,
pkg/datastore/postgres/repositories/block_repository_test.go
+21 -12
View File
@@ -15,16 +15,18 @@ import (
var _ = Describe("Saving blocks", func() {
var db *postgres.DB
var node core.Node
var blockRepository datastore.BlockRepository
BeforeEach(func() {
node := core.Node{
node = core.Node{
GenesisBlock: "GENESIS",
NetworkID: 1,
ID: "b6f90c0fdd8ec9607aed8ee45c69322e47b7063f0bfb7a29c8ecafab24d0a22d24dd2329b5ee6ed4125a03cb14e57fd584e67f9e53e6c631055cbbd82f080845",
ClientName: "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
}
db = test_config.NewTestDB(node)
blockRepository = repositories.BlockRepository{DB: db}
blockRepository = repositories.NewBlockRepository(db)
})
@@ -40,7 +42,7 @@ var _ = Describe("Saving blocks", func() {
ClientName: "Geth",
}
dbTwo := test_config.NewTestDB(nodeTwo)
repositoryTwo := repositories.BlockRepository{DB: dbTwo}
repositoryTwo := repositories.NewBlockRepository(dbTwo)
_, err := repositoryTwo.GetBlock(123)
Expect(err).To(HaveOccurred())
@@ -166,7 +168,7 @@ var _ = Describe("Saving blocks", func() {
NetworkID: 1,
}
dbTwo := test_config.NewTestDB(nodeTwo)
repositoryTwo := repositories.BlockRepository{DB: dbTwo}
repositoryTwo := repositories.NewBlockRepository(dbTwo)
blockRepository.CreateOrUpdateBlock(blockOne)
repositoryTwo.CreateOrUpdateBlock(blockTwo)
@@ -236,32 +238,39 @@ var _ = Describe("Saving blocks", func() {
It("is empty the starting block number is the highest known block number", func() {
blockRepository.CreateOrUpdateBlock(core.Block{Number: 1})
Expect(len(blockRepository.MissingBlockNumbers(1, 1))).To(Equal(0))
Expect(len(blockRepository.MissingBlockNumbers(1, 1, node.ID))).To(Equal(0))
})
It("is the only missing block number", func() {
blockRepository.CreateOrUpdateBlock(core.Block{Number: 2})
Expect(blockRepository.MissingBlockNumbers(1, 2)).To(Equal([]int64{1}))
Expect(blockRepository.MissingBlockNumbers(1, 2, node.ID)).To(Equal([]int64{1}))
})
It("is both missing block numbers", func() {
blockRepository.CreateOrUpdateBlock(core.Block{Number: 3})
Expect(blockRepository.MissingBlockNumbers(1, 3)).To(Equal([]int64{1, 2}))
Expect(blockRepository.MissingBlockNumbers(1, 3, node.ID)).To(Equal([]int64{1, 2}))
})
It("goes back to the starting block number", func() {
blockRepository.CreateOrUpdateBlock(core.Block{Number: 6})
Expect(blockRepository.MissingBlockNumbers(4, 6)).To(Equal([]int64{4, 5}))
Expect(blockRepository.MissingBlockNumbers(4, 6, node.ID)).To(Equal([]int64{4, 5}))
})
It("only includes missing block numbers", func() {
blockRepository.CreateOrUpdateBlock(core.Block{Number: 4})
blockRepository.CreateOrUpdateBlock(core.Block{Number: 6})
Expect(blockRepository.MissingBlockNumbers(4, 6)).To(Equal([]int64{5}))
Expect(blockRepository.MissingBlockNumbers(4, 6, node.ID)).To(Equal([]int64{5}))
})
It("includes blocks created by a different node", func() {
blockRepository.CreateOrUpdateBlock(core.Block{Number: 4})
blockRepository.CreateOrUpdateBlock(core.Block{Number: 6})
Expect(blockRepository.MissingBlockNumbers(4, 6, "Different node id")).To(Equal([]int64{4, 5, 6}))
})
It("is a list with multiple gaps", func() {
@@ -270,18 +279,18 @@ var _ = Describe("Saving blocks", func() {
blockRepository.CreateOrUpdateBlock(core.Block{Number: 8})
blockRepository.CreateOrUpdateBlock(core.Block{Number: 10})
Expect(blockRepository.MissingBlockNumbers(3, 10)).To(Equal([]int64{3, 6, 7, 9}))
Expect(blockRepository.MissingBlockNumbers(3, 10, node.ID)).To(Equal([]int64{3, 6, 7, 9}))
})
It("returns empty array when lower bound exceeds upper bound", func() {
Expect(blockRepository.MissingBlockNumbers(10000, 1)).To(Equal([]int64{}))
Expect(blockRepository.MissingBlockNumbers(10000, 1, node.ID)).To(Equal([]int64{}))
})
It("only returns requested range even when other gaps exist", func() {
blockRepository.CreateOrUpdateBlock(core.Block{Number: 3})
blockRepository.CreateOrUpdateBlock(core.Block{Number: 8})
Expect(blockRepository.MissingBlockNumbers(1, 5)).To(Equal([]int64{1, 2, 4, 5}))
Expect(blockRepository.MissingBlockNumbers(1, 5, node.ID)).To(Equal([]int64{1, 2, 4, 5}))
})
})
pkg/datastore/postgres/repositories/contract_repository_test.go
+1 -1
View File
@@ -53,7 +53,7 @@ var _ = Describe("Creating contracts", func() {
It("returns transactions 'To' a contract", func() {
var blockRepository datastore.BlockRepository
blockRepository = repositories.BlockRepository{DB: db}
blockRepository = repositories.NewBlockRepository(db)
block := core.Block{
Number: 123,
Transactions: []core.Transaction{
pkg/datastore/postgres/repositories/logs_repository_test.go
+1 -1
View File
@@ -28,7 +28,7 @@ var _ = Describe("Logs Repository", func() {
ClientName: "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
}
db = test_config.NewTestDB(node)
blockRepository = repositories.BlockRepository{DB: db}
blockRepository = repositories.NewBlockRepository(db)
logsRepository = repositories.LogRepository{DB: db}
receiptRepository = repositories.ReceiptRepository{DB: db}
})
pkg/datastore/postgres/repositories/receipts_repository_test.go
+9 -7
View File
@@ -10,7 +10,7 @@ import (
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Logs Repository", func() {
var _ = Describe("Receipts Repository", func() {
var blockRepository datastore.BlockRepository
var logRepository datastore.LogRepository
var receiptRepository datastore.ReceiptRepository
@@ -24,7 +24,7 @@ var _ = Describe("Logs Repository", func() {
ClientName: "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
}
db = test_config.NewTestDB(node)
blockRepository = repositories.BlockRepository{DB: db}
blockRepository = repositories.NewBlockRepository(db)
logRepository = repositories.LogRepository{DB: db}
receiptRepository = repositories.ReceiptRepository{DB: db}
})
@@ -99,11 +99,12 @@ var _ = Describe("Logs Repository", func() {
Hash: expected.TxHash,
Receipt: expected,
}
block := core.Block{Transactions: []core.Transaction{transaction}}
blockRepository.CreateOrUpdateBlock(block)
receipt, err := receiptRepository.GetReceipt("0xe340558980f89d5f86045ac11e5cc34e4bcec20f9f1e2a427aa39d87114e8223")
_, err := blockRepository.CreateOrUpdateBlock(block)
Expect(err).NotTo(HaveOccurred())
receipt, err := receiptRepository.GetReceipt("0xe340558980f89d5f86045ac11e5cc34e4bcec20f9f1e2a427aa39d87114e8223")
Expect(err).ToNot(HaveOccurred())
//Not currently serializing bloom logs
Expect(receipt.Bloom).To(Equal(core.Receipt{}.Bloom))
@@ -132,10 +133,11 @@ var _ = Describe("Logs Repository", func() {
block := core.Block{
Transactions: []core.Transaction{transaction},
}
blockRepository.CreateOrUpdateBlock(block)
_, err := receiptRepository.GetReceipt(receipt.TxHash)
_, err := blockRepository.CreateOrUpdateBlock(block)
Expect(err).NotTo(HaveOccurred())
_, err = receiptRepository.GetReceipt(receipt.TxHash)
Expect(err).To(Not(HaveOccurred()))
})
})
pkg/datastore/postgres/repositories/watched_events_repository_test.go
+1 -1
View File
@@ -21,7 +21,7 @@ var _ = Describe("Watched Events Repository", func() {
BeforeEach(func() {
db = test_config.NewTestDB(core.Node{})
blocksRepository = repositories.BlockRepository{DB: db}
blocksRepository = repositories.NewBlockRepository(db)
filterRepository = repositories.FilterRepository{DB: db}
logRepository = repositories.LogRepository{DB: db}
receiptRepository = repositories.ReceiptRepository{DB: db}
pkg/datastore/repository.go
+1 -1
View File
@@ -14,7 +14,7 @@ var ErrBlockDoesNotExist = func(blockNumber int64) error {
type BlockRepository interface {
CreateOrUpdateBlock(block core.Block) (int64, error)
GetBlock(blockNumber int64) (core.Block, error)
MissingBlockNumbers(startingBlockNumber int64, endingBlockNumber int64) []int64
MissingBlockNumbers(startingBlockNumber int64, endingBlockNumber int64, nodeId string) []int64
SetBlocksStatus(chainHead int64)
}