reorg pkg/ to prepare to support chains other than ethereumm

This commit is contained in:
Ian Norden
2020-02-20 16:14:16 -06:00
parent 1412f5a7f5
commit da844b0b83
240 changed files with 440 additions and 440 deletions
+65
View File
@@ -0,0 +1,65 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package history
import (
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/eth/core"
"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
)
type BlockValidator struct {
blockchain core.BlockChain
blockRepository datastore.BlockRepository
windowSize int
}
func NewBlockValidator(blockchain core.BlockChain, blockRepository datastore.BlockRepository, windowSize int) *BlockValidator {
return &BlockValidator{
blockchain: blockchain,
blockRepository: blockRepository,
windowSize: windowSize,
}
}
func (bv BlockValidator) ValidateBlocks() (ValidationWindow, error) {
window, err := MakeValidationWindow(bv.blockchain, bv.windowSize)
if err != nil {
logrus.Error("ValidateBlocks: error creating validation window: ", err)
return ValidationWindow{}, err
}
blockNumbers := MakeRange(window.LowerBound, window.UpperBound)
_, err = RetrieveAndUpdateBlocks(bv.blockchain, bv.blockRepository, blockNumbers)
if err != nil {
logrus.Error("ValidateBlocks: error getting and updating blocks: ", err)
return ValidationWindow{}, err
}
lastBlock, err := bv.blockchain.LastBlock()
if err != nil {
logrus.Error("ValidateBlocks: error getting last block: ", err)
return ValidationWindow{}, err
}
err = bv.blockRepository.SetBlocksStatus(lastBlock.Int64())
if err != nil {
logrus.Error("ValidateBlocks: error setting block status: ", err)
return ValidationWindow{}, err
}
return window, nil
}
+51
View File
@@ -0,0 +1,51 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package history_test
import (
"math/big"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/eth/history"
"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
)
var _ = Describe("Blocks validator", func() {
It("calls create or update for all blocks within the window", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(7))
blocksRepository := fakes.NewMockBlockRepository()
validator := history.NewBlockValidator(blockChain, blocksRepository, 2)
window, err := validator.ValidateBlocks()
Expect(err).NotTo(HaveOccurred())
Expect(window).To(Equal(history.ValidationWindow{LowerBound: 5, UpperBound: 7}))
blocksRepository.AssertCreateOrUpdateBlockCallCountEquals(3)
})
It("returns the number of largest block", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(3))
maxBlockNumber, _ := blockChain.LastBlock()
Expect(maxBlockNumber.Int64()).To(Equal(int64(3)))
})
})
+52
View File
@@ -0,0 +1,52 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package history
import (
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/eth/core"
"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
)
type HeaderValidator struct {
blockChain core.BlockChain
headerRepository datastore.HeaderRepository
windowSize int
}
func NewHeaderValidator(blockChain core.BlockChain, repository datastore.HeaderRepository, windowSize int) HeaderValidator {
return HeaderValidator{
blockChain: blockChain,
headerRepository: repository,
windowSize: windowSize,
}
}
func (validator HeaderValidator) ValidateHeaders() (ValidationWindow, error) {
window, err := MakeValidationWindow(validator.blockChain, validator.windowSize)
if err != nil {
logrus.Error("ValidateHeaders: error creating validation window: ", err)
return ValidationWindow{}, err
}
blockNumbers := MakeRange(window.LowerBound, window.UpperBound)
_, err = RetrieveAndUpdateHeaders(validator.blockChain, validator.headerRepository, blockNumbers)
if err != nil {
logrus.Error("ValidateHeaders: error getting/updating headers: ", err)
return ValidationWindow{}, err
}
return window, nil
}
+61
View File
@@ -0,0 +1,61 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package history_test
import (
"errors"
"math/big"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/eth/history"
"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
)
var _ = Describe("Header validator", func() {
var (
headerRepository *fakes.MockHeaderRepository
blockChain *fakes.MockBlockChain
)
BeforeEach(func() {
headerRepository = fakes.NewMockHeaderRepository()
blockChain = fakes.NewMockBlockChain()
})
It("attempts to create every header in the validation window", func() {
headerRepository.SetMissingBlockNumbers([]int64{})
blockChain.SetLastBlock(big.NewInt(3))
validator := history.NewHeaderValidator(blockChain, headerRepository, 2)
_, err := validator.ValidateHeaders()
Expect(err).NotTo(HaveOccurred())
headerRepository.AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(3, []int64{1, 2, 3})
})
It("propagates header repository errors", func() {
blockChain.SetLastBlock(big.NewInt(3))
headerRepositoryError := errors.New("CreateOrUpdate")
headerRepository.SetCreateOrUpdateHeaderReturnErr(headerRepositoryError)
validator := history.NewHeaderValidator(blockChain, headerRepository, 2)
_, err := validator.ValidateHeaders()
Expect(err).To(MatchError(headerRepositoryError))
})
})
+35
View File
@@ -0,0 +1,35 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package history_test
import (
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
log "github.com/sirupsen/logrus"
)
func init() {
log.SetOutput(ioutil.Discard)
}
func TestHistory(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "History Suite")
}
+69
View File
@@ -0,0 +1,69 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package history
import (
"fmt"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/eth/core"
"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
)
func PopulateMissingBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, startingBlockNumber int64) (int, error) {
lastBlock, err := blockchain.LastBlock()
if err != nil {
log.Error("PopulateMissingBlocks: error getting last block: ", err)
return 0, err
}
blockRange := blockRepository.MissingBlockNumbers(startingBlockNumber, lastBlock.Int64(), blockchain.Node().ID)
if len(blockRange) == 0 {
return 0, nil
}
log.Debug(getBlockRangeString(blockRange))
_, err = RetrieveAndUpdateBlocks(blockchain, blockRepository, blockRange)
if err != nil {
log.Error("PopulateMissingBlocks: error gettings/updating blocks: ", err)
return 0, err
}
return len(blockRange), nil
}
func RetrieveAndUpdateBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, blockNumbers []int64) (int, error) {
for _, blockNumber := range blockNumbers {
block, err := blockchain.GetBlockByNumber(blockNumber)
if err != nil {
log.Error("RetrieveAndUpdateBlocks: error getting block: ", err)
return 0, err
}
_, err = blockRepository.CreateOrUpdateBlock(block)
if err != nil {
log.Error("RetrieveAndUpdateBlocks: error creating/updating block: ", err)
return 0, err
}
}
return len(blockNumbers), nil
}
func getBlockRangeString(blockRange []int64) string {
return fmt.Sprintf("Backfilling |%v| blocks", len(blockRange))
}
+90
View File
@@ -0,0 +1,90 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package history_test
import (
"math/big"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/eth/history"
"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
)
var _ = Describe("Populating blocks", func() {
var blockRepository *fakes.MockBlockRepository
BeforeEach(func() {
blockRepository = fakes.NewMockBlockRepository()
})
It("fills in the only missing block (BlockNumber 1)", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(2))
blockRepository.SetMissingBlockNumbersReturnArray([]int64{2})
blocksAdded, err := history.PopulateMissingBlocks(blockChain, blockRepository, 1)
Expect(err).NotTo(HaveOccurred())
_, err = blockRepository.GetBlock(1)
Expect(blocksAdded).To(Equal(1))
Expect(err).ToNot(HaveOccurred())
})
It("fills in the three missing blocks (Numbers: 5,8,10)", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(13))
blockRepository.SetMissingBlockNumbersReturnArray([]int64{5, 8, 10})
blocksAdded, err := history.PopulateMissingBlocks(blockChain, blockRepository, 5)
Expect(err).NotTo(HaveOccurred())
Expect(blocksAdded).To(Equal(3))
blockRepository.AssertCreateOrUpdateBlocksCallCountAndBlockNumbersEquals(3, []int64{5, 8, 10})
})
It("returns the number of blocks created", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(6))
blockRepository.SetMissingBlockNumbersReturnArray([]int64{4, 5})
numberOfBlocksCreated, err := history.PopulateMissingBlocks(blockChain, blockRepository, 3)
Expect(err).NotTo(HaveOccurred())
Expect(numberOfBlocksCreated).To(Equal(2))
})
It("updates the repository with a range of blocks w/in the range ", func() {
blockChain := fakes.NewMockBlockChain()
_, err := history.RetrieveAndUpdateBlocks(blockChain, blockRepository, history.MakeRange(2, 5))
Expect(err).NotTo(HaveOccurred())
blockRepository.AssertCreateOrUpdateBlocksCallCountAndBlockNumbersEquals(4, []int64{2, 3, 4, 5})
})
It("does not call repository create block when there is an error", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetGetBlockByNumberErr(fakes.FakeError)
blocks := history.MakeRange(1, 10)
_, err := history.RetrieveAndUpdateBlocks(blockChain, blockRepository, blocks)
Expect(err).To(HaveOccurred())
blockRepository.AssertCreateOrUpdateBlockCallCountEquals(0)
})
})
+63
View File
@@ -0,0 +1,63 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package history
import (
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/eth/core"
"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
)
func PopulateMissingHeaders(blockChain core.BlockChain, headerRepository datastore.HeaderRepository, startingBlockNumber int64) (int, error) {
lastBlock, err := blockChain.LastBlock()
if err != nil {
logrus.Error("PopulateMissingHeaders: Error getting last block: ", err)
return 0, err
}
blockNumbers, err := headerRepository.MissingBlockNumbers(startingBlockNumber, lastBlock.Int64(), blockChain.Node().ID)
if err != nil {
logrus.Error("PopulateMissingHeaders: Error getting missing block numbers: ", err)
return 0, err
} else if len(blockNumbers) == 0 {
return 0, nil
}
logrus.Debug(getBlockRangeString(blockNumbers))
_, err = RetrieveAndUpdateHeaders(blockChain, headerRepository, blockNumbers)
if err != nil {
logrus.Error("PopulateMissingHeaders: Error getting/updating headers: ", err)
return 0, err
}
return len(blockNumbers), nil
}
func RetrieveAndUpdateHeaders(blockChain core.BlockChain, headerRepository datastore.HeaderRepository, blockNumbers []int64) (int, error) {
headers, err := blockChain.GetHeadersByNumbers(blockNumbers)
for _, header := range headers {
_, err = headerRepository.CreateOrUpdateHeader(header)
if err != nil {
if err == repositories.ErrValidHeaderExists {
continue
}
return 0, err
}
}
return len(blockNumbers), nil
}
+67
View File
@@ -0,0 +1,67 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package history_test
import (
"math/big"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/eth/history"
"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
)
var _ = Describe("Populating headers", func() {
var headerRepository *fakes.MockHeaderRepository
BeforeEach(func() {
headerRepository = fakes.NewMockHeaderRepository()
})
It("returns number of headers added", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(2))
headerRepository.SetMissingBlockNumbers([]int64{2})
headersAdded, err := history.PopulateMissingHeaders(blockChain, headerRepository, 1)
Expect(err).NotTo(HaveOccurred())
Expect(headersAdded).To(Equal(1))
})
It("adds missing headers to the db", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(2))
headerRepository.SetMissingBlockNumbers([]int64{2})
_, err := history.PopulateMissingHeaders(blockChain, headerRepository, 1)
Expect(err).NotTo(HaveOccurred())
headerRepository.AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(1, []int64{2})
})
It("returns early if the db is already synced up to the head of the chain", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(2))
headersAdded, err := history.PopulateMissingHeaders(blockChain, headerRepository, 2)
Expect(err).NotTo(HaveOccurred())
Expect(headersAdded).To(Equal(0))
})
})
+57
View File
@@ -0,0 +1,57 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package history
import (
"fmt"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/eth/core"
)
type ValidationWindow struct {
LowerBound int64
UpperBound int64
}
func (window ValidationWindow) Size() int {
return int(window.UpperBound - window.LowerBound)
}
func MakeValidationWindow(blockchain core.BlockChain, windowSize int) (ValidationWindow, error) {
upperBound, err := blockchain.LastBlock()
if err != nil {
log.Error("MakeValidationWindow: error getting LastBlock: ", err)
return ValidationWindow{}, err
}
lowerBound := upperBound.Int64() - int64(windowSize)
return ValidationWindow{lowerBound, upperBound.Int64()}, nil
}
func MakeRange(min, max int64) []int64 {
a := make([]int64, max-min+1)
for i := range a {
a[i] = min + int64(i)
}
return a
}
func (window ValidationWindow) GetString() string {
return fmt.Sprintf("Validating Blocks |%v|-- Validation Window --|%v|",
window.LowerBound, window.UpperBound)
}
+53
View File
@@ -0,0 +1,53 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package history_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"math/big"
"github.com/vulcanize/vulcanizedb/pkg/eth/history"
"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
)
var _ = Describe("Validation window", func() {
It("creates a ValidationWindow equal to (HEAD-windowSize, HEAD)", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetLastBlock(big.NewInt(5))
validationWindow, err := history.MakeValidationWindow(blockChain, 2)
Expect(err).NotTo(HaveOccurred())
Expect(validationWindow.LowerBound).To(Equal(int64(3)))
Expect(validationWindow.UpperBound).To(Equal(int64(5)))
})
It("returns the window size", func() {
window := history.ValidationWindow{LowerBound: 1, UpperBound: 3}
Expect(window.Size()).To(Equal(2))
})
It("generates a range of int64s", func() {
numberOfBlocksCreated := history.MakeRange(0, 5)
expected := []int64{0, 1, 2, 3, 4, 5}
Expect(numberOfBlocksCreated).To(Equal(expected))
})
})