Update checked headers for new transformers
- If a header was marked as checked before a transformer was added to the watcher, mark all headers since the new transformer's starting block number as unchecked.
This commit is contained in:
Rob Mulholand
@@ -41,6 +41,12 @@ func (repo CheckedHeadersRepository) MarkHeaderChecked(headerID int64) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove checked_headers rows with block number >= starting block number
|
||||
func (repo CheckedHeadersRepository) MarkHeadersUnchecked(startingBlockNumber int64) error {
|
||||
_, err := repo.db.Exec(`DELETE FROM public.checked_headers WHERE header_id IN (SELECT id FROM public.headers WHERE block_number >= $1)`, startingBlockNumber)
|
||||
return err
|
||||
}
|
||||
|
||||
// Return header_id if not present in checked_headers or its check_count is < passed checkCount
|
||||
func (repo CheckedHeadersRepository) MissingHeaders(startingBlockNumber, endingBlockNumber, checkCount int64) ([]core.Header, error) {
|
||||
var result []core.Header
|
||||
|
||||
@@ -73,6 +73,47 @@ var _ = Describe("Checked Headers repository", func() {
|
||||
})
|
||||
})
|
||||
|
||||
Describe("MarkHeadersUnchecked", func() {
|
||||
It("removes rows for headers <= starting block number", func() {
|
||||
blockNumberOne := rand.Int63()
|
||||
blockNumberTwo := blockNumberOne + 1
|
||||
blockNumberThree := blockNumberOne + 2
|
||||
fakeHeaderOne := fakes.GetFakeHeader(blockNumberOne)
|
||||
fakeHeaderTwo := fakes.GetFakeHeader(blockNumberTwo)
|
||||
fakeHeaderThree := fakes.GetFakeHeader(blockNumberThree)
|
||||
headerRepository := repositories.NewHeaderRepository(db)
|
||||
// insert three headers with incrementing block number
|
||||
headerIdOne, insertHeaderOneErr := headerRepository.CreateOrUpdateHeader(fakeHeaderOne)
|
||||
Expect(insertHeaderOneErr).NotTo(HaveOccurred())
|
||||
headerIdTwo, insertHeaderTwoErr := headerRepository.CreateOrUpdateHeader(fakeHeaderTwo)
|
||||
Expect(insertHeaderTwoErr).NotTo(HaveOccurred())
|
||||
headerIdThree, insertHeaderThreeErr := headerRepository.CreateOrUpdateHeader(fakeHeaderThree)
|
||||
Expect(insertHeaderThreeErr).NotTo(HaveOccurred())
|
||||
// mark all headers checked
|
||||
markHeaderOneCheckedErr := repo.MarkHeaderChecked(headerIdOne)
|
||||
Expect(markHeaderOneCheckedErr).NotTo(HaveOccurred())
|
||||
markHeaderTwoCheckedErr := repo.MarkHeaderChecked(headerIdTwo)
|
||||
Expect(markHeaderTwoCheckedErr).NotTo(HaveOccurred())
|
||||
markHeaderThreeCheckedErr := repo.MarkHeaderChecked(headerIdThree)
|
||||
Expect(markHeaderThreeCheckedErr).NotTo(HaveOccurred())
|
||||
|
||||
// mark headers unchecked since blockNumberTwo
|
||||
err := repo.MarkHeadersUnchecked(blockNumberTwo)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
var headerOneChecked, headerTwoChecked, headerThreeChecked bool
|
||||
getHeaderOneErr := db.Get(&headerOneChecked, `SELECT EXISTS(SELECT 1 FROM public.checked_headers WHERE header_id = $1)`, headerIdOne)
|
||||
Expect(getHeaderOneErr).NotTo(HaveOccurred())
|
||||
Expect(headerOneChecked).To(BeTrue())
|
||||
getHeaderTwoErr := db.Get(&headerTwoChecked, `SELECT EXISTS(SELECT 1 FROM public.checked_headers WHERE header_id = $1)`, headerIdTwo)
|
||||
Expect(getHeaderTwoErr).NotTo(HaveOccurred())
|
||||
Expect(headerTwoChecked).To(BeFalse())
|
||||
getHeaderThreeErr := db.Get(&headerThreeChecked, `SELECT EXISTS(SELECT 1 FROM public.checked_headers WHERE header_id = $1)`, headerIdThree)
|
||||
Expect(getHeaderThreeErr).NotTo(HaveOccurred())
|
||||
Expect(headerThreeChecked).To(BeFalse())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("MissingHeaders", func() {
|
||||
var (
|
||||
headerRepository datastore.HeaderRepository
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
)
|
||||
|
||||
type CheckedLogsRepository struct {
|
||||
db *postgres.DB
|
||||
}
|
||||
|
||||
func NewCheckedLogsRepository(db *postgres.DB) CheckedLogsRepository {
|
||||
return CheckedLogsRepository{db: db}
|
||||
}
|
||||
|
||||
// Return whether a given address + topic0 has been fetched on a previous run of vDB
|
||||
func (repository CheckedLogsRepository) HaveLogsBeenChecked(addresses []string, topic0 string) (bool, error) {
|
||||
for _, address := range addresses {
|
||||
var addressExists bool
|
||||
getAddressExistsErr := repository.db.Get(&addressExists, `SELECT EXISTS(SELECT 1 FROM public.checked_logs WHERE contract_address = $1)`, address)
|
||||
if getAddressExistsErr != nil {
|
||||
return false, getAddressExistsErr
|
||||
}
|
||||
if !addressExists {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
var topicZeroExists bool
|
||||
getTopicZeroExistsErr := repository.db.Get(&topicZeroExists, `SELECT EXISTS(SELECT 1 FROM public.checked_logs WHERE topic_zero = $1)`, topic0)
|
||||
if getTopicZeroExistsErr != nil {
|
||||
return false, getTopicZeroExistsErr
|
||||
}
|
||||
return topicZeroExists, nil
|
||||
}
|
||||
|
||||
// Persist that a given address + topic0 has is being fetched on this run of vDB
|
||||
func (repository CheckedLogsRepository) MarkLogsChecked(addresses []string, topic0 string) error {
|
||||
tx, txErr := repository.db.Beginx()
|
||||
if txErr != nil {
|
||||
return txErr
|
||||
}
|
||||
for _, address := range addresses {
|
||||
_, insertErr := tx.Exec(`INSERT INTO public.checked_logs (contract_address, topic_zero) VALUES ($1, $2)`, address, topic0)
|
||||
if insertErr != nil {
|
||||
rollbackErr := tx.Rollback()
|
||||
if rollbackErr != nil {
|
||||
logrus.Errorf("error rolling back transaction inserting checked logs: %s", rollbackErr.Error())
|
||||
}
|
||||
return insertErr
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package repositories_test
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/fakes"
|
||||
"github.com/vulcanize/vulcanizedb/test_config"
|
||||
)
|
||||
|
||||
var _ = Describe("Checked logs repository", func() {
|
||||
var (
|
||||
db *postgres.DB
|
||||
fakeAddress = fakes.FakeAddress.Hex()
|
||||
fakeAddresses = []string{fakeAddress}
|
||||
fakeTopicZero = fakes.FakeHash.Hex()
|
||||
repository datastore.CheckedLogsRepository
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
db = test_config.NewTestDB(test_config.NewTestNode())
|
||||
test_config.CleanTestDB(db)
|
||||
repository = repositories.NewCheckedLogsRepository(db)
|
||||
})
|
||||
|
||||
Describe("HaveLogsBeenChecked", func() {
|
||||
It("returns true if all addresses and the topic0 are already present in the db", func() {
|
||||
_, insertErr := db.Exec(`INSERT INTO public.checked_logs (contract_address, topic_zero) VALUES ($1, $2)`, fakeAddress, fakeTopicZero)
|
||||
Expect(insertErr).NotTo(HaveOccurred())
|
||||
|
||||
hasBeenChecked, err := repository.HaveLogsBeenChecked(fakeAddresses, fakeTopicZero)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(hasBeenChecked).To(BeTrue())
|
||||
})
|
||||
|
||||
It("returns true if addresses and topic0 were fetched because of a combination of other transformers", func() {
|
||||
anotherFakeAddress := common.HexToAddress("0x" + fakes.RandomString(40)).Hex()
|
||||
anotherFakeTopicZero := common.HexToHash("0x" + fakes.RandomString(64)).Hex()
|
||||
// insert row with matching address but different topic0
|
||||
_, insertOneErr := db.Exec(`INSERT INTO public.checked_logs (contract_address, topic_zero) VALUES ($1, $2)`, fakeAddress, anotherFakeTopicZero)
|
||||
Expect(insertOneErr).NotTo(HaveOccurred())
|
||||
// insert row with matching topic0 but different address
|
||||
_, insertTwoErr := db.Exec(`INSERT INTO public.checked_logs (contract_address, topic_zero) VALUES ($1, $2)`, anotherFakeAddress, fakeTopicZero)
|
||||
Expect(insertTwoErr).NotTo(HaveOccurred())
|
||||
|
||||
hasBeenChecked, err := repository.HaveLogsBeenChecked(fakeAddresses, fakeTopicZero)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(hasBeenChecked).To(BeTrue())
|
||||
})
|
||||
|
||||
It("returns false if any address has not been checked", func() {
|
||||
anotherFakeAddress := common.HexToAddress("0x" + fakes.RandomString(40)).Hex()
|
||||
_, insertErr := db.Exec(`INSERT INTO public.checked_logs (contract_address, topic_zero) VALUES ($1, $2)`, fakeAddress, fakeTopicZero)
|
||||
Expect(insertErr).NotTo(HaveOccurred())
|
||||
|
||||
hasBeenChecked, err := repository.HaveLogsBeenChecked(append(fakeAddresses, anotherFakeAddress), fakeTopicZero)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(hasBeenChecked).To(BeFalse())
|
||||
})
|
||||
|
||||
It("returns false if topic0 has not been checked", func() {
|
||||
anotherFakeTopicZero := common.HexToHash("0x" + fakes.RandomString(64)).Hex()
|
||||
_, insertErr := db.Exec(`INSERT INTO public.checked_logs (contract_address, topic_zero) VALUES ($1, $2)`, fakeAddress, anotherFakeTopicZero)
|
||||
Expect(insertErr).NotTo(HaveOccurred())
|
||||
|
||||
hasBeenChecked, err := repository.HaveLogsBeenChecked(fakeAddresses, fakeTopicZero)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(hasBeenChecked).To(BeFalse())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("MarkLogsChecked", func() {
|
||||
It("adds a row for all of transformer's addresses + topic0", func() {
|
||||
anotherFakeAddress := common.HexToAddress("0x" + fakes.RandomString(40)).Hex()
|
||||
err := repository.MarkLogsChecked(append(fakeAddresses, anotherFakeAddress), fakeTopicZero)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
var comboOneExists, comboTwoExists bool
|
||||
getComboOneErr := db.Get(&comboOneExists, `SELECT EXISTS(SELECT 1 FROM public.checked_logs WHERE contract_address = $1 AND topic_zero = $2)`, fakeAddress, fakeTopicZero)
|
||||
Expect(getComboOneErr).NotTo(HaveOccurred())
|
||||
Expect(comboOneExists).To(BeTrue())
|
||||
getComboTwoErr := db.Get(&comboTwoExists, `SELECT EXISTS(SELECT 1 FROM public.checked_logs WHERE contract_address = $1 AND topic_zero = $2)`, anotherFakeAddress, fakeTopicZero)
|
||||
Expect(getComboTwoErr).NotTo(HaveOccurred())
|
||||
Expect(comboTwoExists).To(BeTrue())
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -36,9 +36,15 @@ type BlockRepository interface {
|
||||
|
||||
type CheckedHeadersRepository interface {
|
||||
MarkHeaderChecked(headerID int64) error
|
||||
MarkHeadersUnchecked(startingBlockNumber int64) error
|
||||
MissingHeaders(startingBlockNumber, endingBlockNumber, checkCount int64) ([]core.Header, error)
|
||||
}
|
||||
|
||||
type CheckedLogsRepository interface {
|
||||
HaveLogsBeenChecked(addresses []string, topic0 string) (bool, error)
|
||||
MarkLogsChecked(addresses []string, topic0 string) error
|
||||
}
|
||||
|
||||
type ContractRepository interface {
|
||||
CreateContract(contract core.Contract) error
|
||||
GetContract(contractHash string) (core.Contract, error)
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package fakes
|
||||
|
||||
type MockCheckedLogsRepository struct {
|
||||
HasLogBeenCheckedAddresses []string
|
||||
HasLogBeenCheckedError error
|
||||
HasLogBeenCheckedReturn bool
|
||||
HasLogBeenCheckedTopicZero string
|
||||
MarkLogCheckedAddresses []string
|
||||
MarkLogCheckedError error
|
||||
MarkLogCheckedTopicZero string
|
||||
}
|
||||
|
||||
func (repository *MockCheckedLogsRepository) HaveLogsBeenChecked(addresses []string, topic0 string) (bool, error) {
|
||||
repository.HasLogBeenCheckedAddresses = addresses
|
||||
repository.HasLogBeenCheckedTopicZero = topic0
|
||||
return repository.HasLogBeenCheckedReturn, repository.HasLogBeenCheckedError
|
||||
}
|
||||
|
||||
func (repository *MockCheckedLogsRepository) MarkLogsChecked(addresses []string, topic0 string) error {
|
||||
repository.MarkLogCheckedAddresses = addresses
|
||||
repository.MarkLogCheckedTopicZero = topic0
|
||||
return repository.MarkLogCheckedError
|
||||
}
|
||||
+16
-4
@@ -20,16 +20,16 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strconv"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"math/rand"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
FakeAddress = common.HexToAddress("0x1234567890abcdef")
|
||||
FakeAddress = common.HexToAddress("0x" + RandomString(40))
|
||||
FakeError = errors.New("failed")
|
||||
FakeHash = common.BytesToHash([]byte{1, 2, 3, 4, 5})
|
||||
fakeTimestamp = int64(111111111)
|
||||
@@ -103,3 +103,15 @@ func GetFakeUncle(hash, reward string) core.Uncle {
|
||||
Timestamp: strconv.FormatInt(fakeTimestamp, 10),
|
||||
}
|
||||
}
|
||||
|
||||
func RandomString(length int) string {
|
||||
var seededRand = rand.New(
|
||||
rand.NewSource(time.Now().UnixNano()))
|
||||
charset := "abcdef1234567890"
|
||||
b := make([]byte, length)
|
||||
for i := range b {
|
||||
b[i] = charset[seededRand.Intn(len(charset))]
|
||||
}
|
||||
|
||||
return string(b)
|
||||
}
|
||||
|
||||
@@ -21,23 +21,32 @@ import (
|
||||
)
|
||||
|
||||
type MockCheckedHeadersRepository struct {
|
||||
CheckCount int64
|
||||
StartingBlockNumber int64
|
||||
EndingBlockNumber int64
|
||||
HeaderID int64
|
||||
ReturnHeaders []core.Header
|
||||
MarkHeaderCheckedReturnError error
|
||||
MissingHeadersReturnError error
|
||||
MarkHeaderCheckedHeaderID int64
|
||||
MarkHeaderCheckedReturnError error
|
||||
MarkHeadersUncheckedCalled bool
|
||||
MarkHeadersUncheckedReturnError error
|
||||
MarkHeadersUncheckedStartingBlockNumber int64
|
||||
MissingHeadersCheckCount int64
|
||||
MissingHeadersEndingBlockNumber int64
|
||||
MissingHeadersReturnError error
|
||||
MissingHeadersReturnHeaders []core.Header
|
||||
MissingHeadersStartingBlockNumber int64
|
||||
}
|
||||
|
||||
func (repository *MockCheckedHeadersRepository) MarkHeadersUnchecked(startingBlockNumber int64) error {
|
||||
repository.MarkHeadersUncheckedCalled = true
|
||||
repository.MarkHeadersUncheckedStartingBlockNumber = startingBlockNumber
|
||||
return repository.MarkHeadersUncheckedReturnError
|
||||
}
|
||||
|
||||
func (repository *MockCheckedHeadersRepository) MarkHeaderChecked(headerID int64) error {
|
||||
repository.HeaderID = headerID
|
||||
repository.MarkHeaderCheckedHeaderID = headerID
|
||||
return repository.MarkHeaderCheckedReturnError
|
||||
}
|
||||
|
||||
func (repository *MockCheckedHeadersRepository) MissingHeaders(startingBlockNumber, endingBlockNumber, checkCount int64) ([]core.Header, error) {
|
||||
repository.StartingBlockNumber = startingBlockNumber
|
||||
repository.EndingBlockNumber = endingBlockNumber
|
||||
repository.CheckCount = checkCount
|
||||
return repository.ReturnHeaders, repository.MissingHeadersReturnError
|
||||
repository.MissingHeadersStartingBlockNumber = startingBlockNumber
|
||||
repository.MissingHeadersEndingBlockNumber = endingBlockNumber
|
||||
repository.MissingHeadersCheckCount = checkCount
|
||||
return repository.MissingHeadersReturnHeaders, repository.MissingHeadersReturnError
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user