deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

Add light sync command

Browse Source 
- Only syncs block headers (excludes block bodies, transactions, receipts, and logs)
- Modifies validation window to include the most recent block
- Isolates validation window to the variable defined in the cmd directory (blocks
  have a separate variable defined in the block_repository for determining when
  to set a block as final)
This commit is contained in:
Rob Mulholand
2018-07-18 16:34:12 -05:00
parent a620fc6a51
commit 1355271011
43 changed files with 1092 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/history/block_validator.go
+29
View File
@@ -0,0 +1,29 @@
package history
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
)
type BlockValidator struct {
blockchain core.Blockchain
blockRepository datastore.BlockRepository
windowSize int
}
func NewBlockValidator(blockchain core.Blockchain, blockRepository datastore.BlockRepository, windowSize int) *BlockValidator {
return &BlockValidator{
blockchain: blockchain,
blockRepository: blockRepository,
windowSize: windowSize,
}
}
func (bv BlockValidator) ValidateBlocks() ValidationWindow {
window := MakeValidationWindow(bv.blockchain, bv.windowSize)
blockNumbers := MakeRange(window.LowerBound, window.UpperBound)
RetrieveAndUpdateBlocks(bv.blockchain, bv.blockRepository, blockNumbers)
lastBlock := bv.blockchain.LastBlock().Int64()
bv.blockRepository.SetBlocksStatus(lastBlock)
return window
}
pkg/history/block_validator_test.go
+43
View File
@@ -0,0 +1,43 @@
package history_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/inmemory"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/history"
)
var _ = Describe("Blocks validator", func() {
It("calls create or update for all blocks within the window", func() {
blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
{Number: 4},
{Number: 5},
{Number: 6},
{Number: 7},
})
inMemoryDB := inmemory.NewInMemory()
blocksRepository := &inmemory.BlockRepository{InMemory: inMemoryDB}
validator := history.NewBlockValidator(blockchain, blocksRepository, 2)
window := validator.ValidateBlocks()
Expect(window).To(Equal(history.ValidationWindow{LowerBound: 5, UpperBound: 7}))
Expect(blocksRepository.BlockCount()).To(Equal(3))
Expect(blocksRepository.CreateOrUpdateBlockCallCount).To(Equal(3))
})
It("returns the number of largest block", func() {
blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
{Number: 1},
{Number: 2},
{Number: 3},
})
maxBlockNumber := blockchain.LastBlock()
Expect(maxBlockNumber.Int64()).To(Equal(int64(3)))
})
})
pkg/history/header_validator.go
+27
View File
@@ -0,0 +1,27 @@
package history
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
)
type HeaderValidator struct {
blockChain core.Blockchain
headerRepository datastore.HeaderRepository
windowSize int
}
func NewHeaderValidator(blockChain core.Blockchain, repository datastore.HeaderRepository, windowSize int) HeaderValidator {
return HeaderValidator{
blockChain: blockChain,
headerRepository: repository,
windowSize: windowSize,
}
}
func (validator HeaderValidator) ValidateHeaders() ValidationWindow {
window := MakeValidationWindow(validator.blockChain, validator.windowSize)
blockNumbers := MakeRange(window.LowerBound, window.UpperBound)
RetrieveAndUpdateHeaders(validator.blockChain, validator.headerRepository, blockNumbers)
return window
}
pkg/history/header_validator_test.go
+39
View File
@@ -0,0 +1,39 @@
package history_test
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/inmemory"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/history"
)
var _ = Describe("Header validator", func() {
It("replaces headers in the validation window that have changed", func() {
blockNumber := int64(10)
oldHash := common.HexToHash("0x0987654321").Hex()
oldHeader := core.Header{
BlockNumber: blockNumber,
Hash: oldHash,
}
inMemory := inmemory.NewInMemory()
headerRepository := inmemory.NewHeaderRepository(inMemory)
headerRepository.CreateOrUpdateHeader(oldHeader)
newHash := common.HexToHash("0x123456789").Hex()
newHeader := core.Header{
BlockNumber: blockNumber,
Hash: newHash,
}
headers := []core.Header{newHeader}
blockChain := fakes.NewBlockChainWithHeaders(headers)
validator := history.NewHeaderValidator(blockChain, headerRepository, 1)
validator.ValidateHeaders()
dbHeader, _ := headerRepository.GetHeader(blockNumber)
Expect(dbHeader.Hash).NotTo(Equal(oldHash))
Expect(dbHeader.Hash).To(Equal(newHash))
})
})
pkg/history/populate_blocks.go
+1 -1
View File
@@ -9,7 +9,7 @@ import (
func PopulateMissingBlocks(blockchain core.Blockchain, blockRepository datastore.BlockRepository, startingBlockNumber int64) int {
lastBlock := blockchain.LastBlock().Int64()
blockRange := blockRepository.MissingBlockNumbers(startingBlockNumber, lastBlock-1, blockchain.Node().ID)
blockRange := blockRepository.MissingBlockNumbers(startingBlockNumber, lastBlock, blockchain.Node().ID)
log.SetPrefix("")
log.Printf("Backfilling %d blocks\n\n", len(blockRange))
RetrieveAndUpdateBlocks(blockchain, blockRepository, blockRange)
pkg/history/populate_blocks_test.go
+6 -6
View File
@@ -20,7 +20,7 @@ var _ = Describe("Populating blocks", func() {
blockRepository = &inmemory.BlockRepository{InMemory: inMemory}
})
It("fills in the only missing block (Number 1)", func() {
It("fills in the only missing block (BlockNumber 1)", func() {
blocks := []core.Block{
{Number: 1},
{Number: 2},
@@ -57,11 +57,12 @@ var _ = Describe("Populating blocks", func() {
blockRepository.CreateOrUpdateBlock(core.Block{Number: 9})
blockRepository.CreateOrUpdateBlock(core.Block{Number: 11})
blockRepository.CreateOrUpdateBlock(core.Block{Number: 12})
blockRepository.CreateOrUpdateBlock(core.Block{Number: 13})
blocksAdded := history.PopulateMissingBlocks(blockchain, blockRepository, 5)
Expect(blocksAdded).To(Equal(3))
Expect(blockRepository.BlockCount()).To(Equal(11))
Expect(blockRepository.BlockCount()).To(Equal(12))
_, err := blockRepository.GetBlock(4)
Expect(err).To(HaveOccurred())
_, err = blockRepository.GetBlock(5)
@@ -70,7 +71,7 @@ var _ = Describe("Populating blocks", func() {
Expect(err).ToNot(HaveOccurred())
_, err = blockRepository.GetBlock(10)
Expect(err).ToNot(HaveOccurred())
_, err = blockRepository.GetBlock(13)
_, err = blockRepository.GetBlock(14)
Expect(err).To(HaveOccurred())
})
@@ -98,8 +99,8 @@ var _ = Describe("Populating blocks", func() {
})
history.RetrieveAndUpdateBlocks(blockchain, blockRepository, history.MakeRange(2, 5))
Expect(blockRepository.BlockCount()).To(Equal(3))
Expect(blockRepository.CreateOrUpdateBlockCallCount).To(Equal(3))
Expect(blockRepository.BlockCount()).To(Equal(4))
Expect(blockRepository.CreateOrUpdateBlockCallCount).To(Equal(4))
})
It("does not call repository create block when there is an error", func() {
@@ -109,5 +110,4 @@ var _ = Describe("Populating blocks", func() {
Expect(blockRepository.BlockCount()).To(Equal(0))
Expect(blockRepository.CreateOrUpdateBlockCallCount).To(Equal(0))
})
})
pkg/history/populate_headers.go
+29
View File
@@ -0,0 +1,29 @@
package history
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"log"
)
func PopulateMissingHeaders(blockchain core.Blockchain, headerRepository datastore.HeaderRepository, startingBlockNumber int64) int {
lastBlock := blockchain.LastBlock().Int64()
blockRange := headerRepository.MissingBlockNumbers(startingBlockNumber, lastBlock, blockchain.Node().ID)
log.SetPrefix("")
log.Printf("Backfilling %d blocks\n\n", len(blockRange))
RetrieveAndUpdateHeaders(blockchain, headerRepository, blockRange)
return len(blockRange)
}
func RetrieveAndUpdateHeaders(blockchain core.Blockchain, headerRepository datastore.HeaderRepository, blockNumbers []int64) int {
for _, blockNumber := range blockNumbers {
header, err := blockchain.GetHeaderByNumber(blockNumber)
if err != nil {
log.Printf("failed to retrieve block number: %d\n", blockNumber)
return 0
}
// TODO: handle possible error here
headerRepository.CreateOrUpdateHeader(header)
}
return len(blockNumbers)
}
pkg/history/populate_headers_test.go
+52
View File
@@ -0,0 +1,52 @@
package history_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/inmemory"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/history"
)
var _ = Describe("Populating headers", func() {
var inMemory *inmemory.InMemory
var headerRepository *inmemory.HeaderRepository
BeforeEach(func() {
inMemory = inmemory.NewInMemory()
headerRepository = inmemory.NewHeaderRepository(inMemory)
})
Describe("When 1 missing header", func() {
It("returns number of headers added", func() {
headers := []core.Header{
{BlockNumber: 1},
{BlockNumber: 2},
}
blockChain := fakes.NewBlockChainWithHeaders(headers)
headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: 2})
headersAdded := history.PopulateMissingHeaders(blockChain, headerRepository, 1)
Expect(headersAdded).To(Equal(1))
})
})
It("adds missing headers to the db", func() {
headers := []core.Header{
{BlockNumber: 1},
{BlockNumber: 2},
}
blockChain := fakes.NewBlockChainWithHeaders(headers)
dbHeader, _ := headerRepository.GetHeader(1)
Expect(dbHeader.BlockNumber).To(BeZero())
history.PopulateMissingHeaders(blockChain, headerRepository, 1)
dbHeader, _ = headerRepository.GetHeader(1)
Expect(dbHeader.BlockNumber).To(Equal(int64(1)))
})
})
pkg/history/validate_blocks.go
-68
View File
@@ -1,68 +0,0 @@
package history
import (
"io"
"text/template"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
)
const WindowTemplate = `Validating Blocks
|{{.LowerBound}}|-- Validation Window --|{{.UpperBound}}| ({{.UpperBound}}:HEAD)
`
var ParsedWindowTemplate = *template.Must(template.New("window").Parse(WindowTemplate))
type BlockValidator struct {
blockchain core.Blockchain
blockRepository datastore.BlockRepository
windowSize int
parsedLoggingTemplate template.Template
}
func NewBlockValidator(blockchain core.Blockchain, blockRepository datastore.BlockRepository, windowSize int) *BlockValidator {
return &BlockValidator{
blockchain,
blockRepository,
windowSize,
ParsedWindowTemplate,
}
}
func (bv BlockValidator) ValidateBlocks() ValidationWindow {
window := MakeValidationWindow(bv.blockchain, bv.windowSize)
blockNumbers := MakeRange(window.LowerBound, window.UpperBound)
RetrieveAndUpdateBlocks(bv.blockchain, bv.blockRepository, blockNumbers)
lastBlock := bv.blockchain.LastBlock().Int64()
bv.blockRepository.SetBlocksStatus(lastBlock)
return window
}
func (bv BlockValidator) Log(out io.Writer, window ValidationWindow) {
bv.parsedLoggingTemplate.Execute(out, window)
}
type ValidationWindow struct {
LowerBound int64
UpperBound int64
}
func (window ValidationWindow) Size() int {
return int(window.UpperBound - window.LowerBound)
}
func MakeValidationWindow(blockchain core.Blockchain, windowSize int) ValidationWindow {
upperBound := blockchain.LastBlock().Int64()
lowerBound := upperBound - int64(windowSize)
return ValidationWindow{lowerBound, upperBound}
}
func MakeRange(min, max int64) []int64 {
a := make([]int64, max-min)
for i := range a {
a[i] = min + int64(i)
}
return a
}
pkg/history/validate_blocks_test.go
-86
View File
@@ -1,86 +0,0 @@
package history_test
import (
"bytes"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/inmemory"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/history"
)
var _ = Describe("Blocks validator", func() {
It("creates a ValidationWindow equal to (HEAD-windowSize, HEAD)", func() {
blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
{Number: 1},
{Number: 2},
{Number: 3},
{Number: 4},
{Number: 5},
})
validationWindow := history.MakeValidationWindow(blockchain, 2)
Expect(validationWindow.LowerBound).To(Equal(int64(3)))
Expect(validationWindow.UpperBound).To(Equal(int64(5)))
})
It("returns the window size", func() {
window := history.ValidationWindow{LowerBound: 1, UpperBound: 3}
Expect(window.Size()).To(Equal(2))
})
It("calls create or update for all blocks within the window", func() {
blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
{Number: 4},
{Number: 5},
{Number: 6},
{Number: 7},
})
inMemoryDB := inmemory.NewInMemory()
blocksRepository := &inmemory.BlockRepository{InMemory: inMemoryDB}
validator := history.NewBlockValidator(blockchain, blocksRepository, 2)
window := validator.ValidateBlocks()
Expect(window).To(Equal(history.ValidationWindow{LowerBound: 5, UpperBound: 7}))
Expect(blocksRepository.BlockCount()).To(Equal(2))
Expect(blocksRepository.CreateOrUpdateBlockCallCount).To(Equal(2))
})
It("logs window message", func() {
inMemoryDB := inmemory.NewInMemory()
blockRepository := &inmemory.BlockRepository{InMemory: inMemoryDB}
expectedMessage := &bytes.Buffer{}
window := history.ValidationWindow{LowerBound: 5, UpperBound: 7}
history.ParsedWindowTemplate.Execute(expectedMessage, history.ValidationWindow{LowerBound: 5, UpperBound: 7})
blockchain := fakes.NewBlockchainWithBlocks([]core.Block{})
validator := history.NewBlockValidator(blockchain, blockRepository, 2)
actualMessage := &bytes.Buffer{}
validator.Log(actualMessage, window)
Expect(actualMessage).To(Equal(expectedMessage))
})
It("generates a range of int64s", func() {
numberOfBlocksCreated := history.MakeRange(0, 5)
expected := []int64{0, 1, 2, 3, 4}
Expect(numberOfBlocksCreated).To(Equal(expected))
})
It("returns the number of largest block", func() {
blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
{Number: 1},
{Number: 2},
{Number: 3},
})
maxBlockNumber := blockchain.LastBlock()
Expect(maxBlockNumber.Int64()).To(Equal(int64(3)))
})
})
pkg/history/validation_window.go
+41
View File
@@ -0,0 +1,41 @@
package history
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"io"
"text/template"
)
const WindowTemplate = `Validating Blocks
|{{.LowerBound}}|-- Validation Window --|{{.UpperBound}}| ({{.UpperBound}}:HEAD)
`
var ParsedWindowTemplate = *template.Must(template.New("window").Parse(WindowTemplate))
type ValidationWindow struct {
LowerBound int64
UpperBound int64
}
func (window ValidationWindow) Size() int {
return int(window.UpperBound - window.LowerBound)
}
func MakeValidationWindow(blockchain core.Blockchain, windowSize int) ValidationWindow {
upperBound := blockchain.LastBlock().Int64()
lowerBound := upperBound - int64(windowSize)
return ValidationWindow{lowerBound, upperBound}
}
func MakeRange(min, max int64) []int64 {
a := make([]int64, max-min+1)
for i := range a {
a[i] = min + int64(i)
}
return a
}
func (window ValidationWindow) Log(out io.Writer) {
ParsedWindowTemplate.Execute(out, window)
}
pkg/history/validation_window_test.go
+53
View File
@@ -0,0 +1,53 @@
package history_test
import (
"bytes"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/history"
)
var _ = Describe("", func() {
It("creates a ValidationWindow equal to (HEAD-windowSize, HEAD)", func() {
blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
{Number: 1},
{Number: 2},
{Number: 3},
{Number: 4},
{Number: 5},
})
validationWindow := history.MakeValidationWindow(blockchain, 2)
Expect(validationWindow.LowerBound).To(Equal(int64(3)))
Expect(validationWindow.UpperBound).To(Equal(int64(5)))
})
It("returns the window size", func() {
window := history.ValidationWindow{LowerBound: 1, UpperBound: 3}
Expect(window.Size()).To(Equal(2))
})
It("generates a range of int64s", func() {
numberOfBlocksCreated := history.MakeRange(0, 5)
expected := []int64{0, 1, 2, 3, 4, 5}
Expect(numberOfBlocksCreated).To(Equal(expected))
})
It("logs window message", func() {
expectedMessage := &bytes.Buffer{}
window := history.ValidationWindow{LowerBound: 5, UpperBound: 7}
history.ParsedWindowTemplate.Execute(expectedMessage, window)
actualMessage := &bytes.Buffer{}
window.Log(actualMessage)
Expect(actualMessage).To(Equal(expectedMessage))
})
})