deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

major refactor part 2: remove cold import, full sync, generalize node table

Browse Source
This commit is contained in:
Ian Norden
2020-02-20 16:14:17 -06:00
parent 0f765df12c
commit ca273a026d
171 changed files with 196 additions and 6330 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/eth/history/block_validator.go
-65
View File
@@ -1,65 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package history
import (
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/eth/core"
"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
)
type BlockValidator struct {
blockchain core.BlockChain
blockRepository datastore.BlockRepository
windowSize int
}
func NewBlockValidator(blockchain core.BlockChain, blockRepository datastore.BlockRepository, windowSize int) *BlockValidator {
return &BlockValidator{
blockchain: blockchain,
blockRepository: blockRepository,
windowSize: windowSize,
}
}
func (bv BlockValidator) ValidateBlocks() (ValidationWindow, error) {
window, err := MakeValidationWindow(bv.blockchain, bv.windowSize)
if err != nil {
logrus.Error("ValidateBlocks: error creating validation window: ", err)
return ValidationWindow{}, err
}
blockNumbers := MakeRange(window.LowerBound, window.UpperBound)
_, err = RetrieveAndUpdateBlocks(bv.blockchain, bv.blockRepository, blockNumbers)
if err != nil {
logrus.Error("ValidateBlocks: error getting and updating blocks: ", err)
return ValidationWindow{}, err
}
lastBlock, err := bv.blockchain.LastBlock()
if err != nil {
logrus.Error("ValidateBlocks: error getting last block: ", err)
return ValidationWindow{}, err
}
err = bv.blockRepository.SetBlocksStatus(lastBlock.Int64())
if err != nil {
logrus.Error("ValidateBlocks: error setting block status: ", err)
return ValidationWindow{}, err
}
return window, nil
}
pkg/eth/history/block_validator_test.go
-51
View File
@@ -1,51 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package history_test
import (
"math/big"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
"github.com/vulcanize/vulcanizedb/pkg/eth/history"
)
var _ = Describe("Blocks validator", func() {
It("calls create or update for all blocks within the window", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(7))
blocksRepository := fakes.NewMockBlockRepository()
validator := history.NewBlockValidator(blockChain, blocksRepository, 2)
window, err := validator.ValidateBlocks()
Expect(err).NotTo(HaveOccurred())
Expect(window).To(Equal(history.ValidationWindow{LowerBound: 5, UpperBound: 7}))
blocksRepository.AssertCreateOrUpdateBlockCallCountEquals(3)
})
It("returns the number of largest block", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(3))
maxBlockNumber, _ := blockChain.LastBlock()
Expect(maxBlockNumber.Int64()).To(Equal(int64(3)))
})
})
pkg/eth/history/populate_blocks.go
-69
View File
@@ -1,69 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package history
import (
"fmt"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/eth/core"
"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
)
func PopulateMissingBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, startingBlockNumber int64) (int, error) {
lastBlock, err := blockchain.LastBlock()
if err != nil {
log.Error("PopulateMissingBlocks: error getting last block: ", err)
return 0, err
}
blockRange := blockRepository.MissingBlockNumbers(startingBlockNumber, lastBlock.Int64(), blockchain.Node().ID)
if len(blockRange) == 0 {
return 0, nil
}
log.Debug(getBlockRangeString(blockRange))
_, err = RetrieveAndUpdateBlocks(blockchain, blockRepository, blockRange)
if err != nil {
log.Error("PopulateMissingBlocks: error gettings/updating blocks: ", err)
return 0, err
}
return len(blockRange), nil
}
func RetrieveAndUpdateBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, blockNumbers []int64) (int, error) {
for _, blockNumber := range blockNumbers {
block, err := blockchain.GetBlockByNumber(blockNumber)
if err != nil {
log.Error("RetrieveAndUpdateBlocks: error getting block: ", err)
return 0, err
}
_, err = blockRepository.CreateOrUpdateBlock(block)
if err != nil {
log.Error("RetrieveAndUpdateBlocks: error creating/updating block: ", err)
return 0, err
}
}
return len(blockNumbers), nil
}
func getBlockRangeString(blockRange []int64) string {
return fmt.Sprintf("Backfilling |%v| blocks", len(blockRange))
}
pkg/eth/history/populate_blocks_test.go
-90
View File
@@ -1,90 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package history_test
import (
"math/big"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
"github.com/vulcanize/vulcanizedb/pkg/eth/history"
)
var _ = Describe("Populating blocks", func() {
var blockRepository *fakes.MockBlockRepository
BeforeEach(func() {
blockRepository = fakes.NewMockBlockRepository()
})
It("fills in the only missing block (BlockNumber 1)", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(2))
blockRepository.SetMissingBlockNumbersReturnArray([]int64{2})
blocksAdded, err := history.PopulateMissingBlocks(blockChain, blockRepository, 1)
Expect(err).NotTo(HaveOccurred())
_, err = blockRepository.GetBlock(1)
Expect(blocksAdded).To(Equal(1))
Expect(err).ToNot(HaveOccurred())
})
It("fills in the three missing blocks (Numbers: 5,8,10)", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(13))
blockRepository.SetMissingBlockNumbersReturnArray([]int64{5, 8, 10})
blocksAdded, err := history.PopulateMissingBlocks(blockChain, blockRepository, 5)
Expect(err).NotTo(HaveOccurred())
Expect(blocksAdded).To(Equal(3))
blockRepository.AssertCreateOrUpdateBlocksCallCountAndBlockNumbersEquals(3, []int64{5, 8, 10})
})
It("returns the number of blocks created", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(6))
blockRepository.SetMissingBlockNumbersReturnArray([]int64{4, 5})
numberOfBlocksCreated, err := history.PopulateMissingBlocks(blockChain, blockRepository, 3)
Expect(err).NotTo(HaveOccurred())
Expect(numberOfBlocksCreated).To(Equal(2))
})
It("updates the repository with a range of blocks w/in the range ", func() {
blockChain := fakes.NewMockBlockChain()
_, err := history.RetrieveAndUpdateBlocks(blockChain, blockRepository, history.MakeRange(2, 5))
Expect(err).NotTo(HaveOccurred())
blockRepository.AssertCreateOrUpdateBlocksCallCountAndBlockNumbersEquals(4, []int64{2, 3, 4, 5})
})
It("does not call repository create block when there is an error", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetGetBlockByNumberErr(fakes.FakeError)
blocks := history.MakeRange(1, 10)
_, err := history.RetrieveAndUpdateBlocks(blockChain, blockRepository, blocks)
Expect(err).To(HaveOccurred())
blockRepository.AssertCreateOrUpdateBlockCallCountEquals(0)
})
})
pkg/eth/history/populate_headers.go
+5
View File
@@ -17,6 +17,7 @@
package history
import (
"fmt"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/eth/core"
@@ -61,3 +62,7 @@ func RetrieveAndUpdateHeaders(blockChain core.BlockChain, headerRepository datas
}
return len(blockNumbers), nil
}
func getBlockRangeString(blockRange []int64) string {
return fmt.Sprintf("Backfilling |%v| blocks", len(blockRange))
}