Decouple log extraction from transformer delegation

- limit missing headers results set to 100 so that extraction doesn't
  excessively block delegation
- wrap checked headers functions in repository struct
- move storage repository to factory, to correspond with event
  repository path
- remove unused files
- reformat sql
- remove line breaks in imports
This commit is contained in:
Rob Mulholand
2019-08-28 09:25:13 -05:00
parent cb819fa9a6
commit d496dad33c
81 changed files with 2049 additions and 1328 deletions
@@ -72,7 +72,7 @@ var _ = Describe("Block Retriever", func() {
Receipt: core.Receipt{
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
ContractAddress: constants.TusdContractAddress,
Logs: []core.Log{},
Logs: []core.FullSyncLog{},
},
TxIndex: 0,
Value: "0",
@@ -92,7 +92,7 @@ var _ = Describe("Block Retriever", func() {
Receipt: core.Receipt{
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
ContractAddress: constants.TusdContractAddress,
Logs: []core.Log{{
Logs: []core.FullSyncLog{{
BlockNumber: 3,
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
Address: constants.TusdContractAddress,
@@ -136,7 +136,7 @@ var _ = Describe("Block Retriever", func() {
Receipt: core.Receipt{
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
ContractAddress: "",
Logs: []core.Log{{
Logs: []core.FullSyncLog{{
BlockNumber: 2,
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
Address: constants.DaiContractAddress,
@@ -167,7 +167,7 @@ var _ = Describe("Block Retriever", func() {
Receipt: core.Receipt{
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
ContractAddress: "",
Logs: []core.Log{{
Logs: []core.FullSyncLog{{
BlockNumber: 3,
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
Address: constants.DaiContractAddress,
@@ -39,7 +39,7 @@ import (
type TransferLog struct {
Id int64 `db:"id"`
VulvanizeLogId int64 `db:"vulcanize_log_id"`
VulcanizeLogId int64 `db:"vulcanize_log_id"`
TokenName string `db:"token_name"`
Block int64 `db:"block"`
Tx string `db:"tx"`
@@ -50,7 +50,7 @@ type TransferLog struct {
type NewOwnerLog struct {
Id int64 `db:"id"`
VulvanizeLogId int64 `db:"vulcanize_log_id"`
VulcanizeLogId int64 `db:"vulcanize_log_id"`
TokenName string `db:"token_name"`
Block int64 `db:"block"`
Tx string `db:"tx"`
@@ -138,13 +138,13 @@ func SetupTusdRepo(vulcanizeLogId *int64, wantedEvents, wantedMethods []string)
Expect(err).NotTo(HaveOccurred())
receiptRepository := repositories.FullSyncReceiptRepository{DB: db}
logRepository := repositories.LogRepository{DB: db}
logRepository := repositories.FullSyncLogRepository{DB: db}
blockRepository := *repositories.NewBlockRepository(db)
blockNumber := rand.Int63()
blockId := CreateBlock(blockNumber, blockRepository)
receipts := []core.Receipt{{Logs: []core.Log{{}}}}
receipts := []core.Receipt{{Logs: []core.FullSyncLog{{}}}}
err = receiptRepository.CreateReceiptsAndLogs(blockId, receipts)
Expect(err).ToNot(HaveOccurred())
@@ -184,13 +184,13 @@ func SetupENSRepo(vulcanizeLogId *int64, wantedEvents, wantedMethods []string) (
Expect(err).NotTo(HaveOccurred())
receiptRepository := repositories.FullSyncReceiptRepository{DB: db}
logRepository := repositories.LogRepository{DB: db}
logRepository := repositories.FullSyncLogRepository{DB: db}
blockRepository := *repositories.NewBlockRepository(db)
blockNumber := rand.Int63()
blockId := CreateBlock(blockNumber, blockRepository)
receipts := []core.Receipt{{Logs: []core.Log{{}}}}
receipts := []core.Receipt{{Logs: []core.FullSyncLog{{}}}}
err = receiptRepository.CreateReceiptsAndLogs(blockId, receipts)
Expect(err).ToNot(HaveOccurred())
@@ -221,6 +221,7 @@ func SetupENSContract(wantedEvents, wantedMethods []string) *contract.Contract {
}.Init()
}
// TODO: tear down/setup DB from migrations so this doesn't alter the schema between tests
func TearDown(db *postgres.DB) {
tx, err := db.Beginx()
Expect(err).NotTo(HaveOccurred())
@@ -255,7 +256,10 @@ func TearDown(db *postgres.DB) {
_, err = tx.Exec(`DROP TABLE checked_headers`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`CREATE TABLE checked_headers (id SERIAL PRIMARY KEY, header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE);`)
_, err = tx.Exec(`CREATE TABLE checked_headers (
id SERIAL PRIMARY KEY,
header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
check_count INTEGER NOT NULL DEFAULT 1);`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)
@@ -41,7 +41,7 @@ var TransferBlock1 = core.Block{
Receipt: core.Receipt{
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654aaa",
ContractAddress: "",
Logs: []core.Log{{
Logs: []core.FullSyncLog{{
BlockNumber: 6194633,
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654aaa",
Address: constants.TusdContractAddress,
@@ -71,7 +71,7 @@ var TransferBlock2 = core.Block{
Receipt: core.Receipt{
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee",
ContractAddress: "",
Logs: []core.Log{{
Logs: []core.FullSyncLog{{
BlockNumber: 6194634,
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee",
Address: constants.TusdContractAddress,
@@ -101,7 +101,7 @@ var NewOwnerBlock1 = core.Block{
Receipt: core.Receipt{
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb",
ContractAddress: "",
Logs: []core.Log{{
Logs: []core.FullSyncLog{{
BlockNumber: 6194635,
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb",
Address: constants.EnsContractAddress,
@@ -131,7 +131,7 @@ var NewOwnerBlock2 = core.Block{
Receipt: core.Receipt{
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654lll",
ContractAddress: "",
Logs: []core.Log{{
Logs: []core.FullSyncLog{{
BlockNumber: 6194636,
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654lll",
Address: constants.EnsContractAddress,
@@ -140,7 +140,7 @@ func (r *eventRepository) persistHeaderSyncLogs(logs []types.Log, eventInfo type
// Mark header as checked for this eventId
eventId := strings.ToLower(eventInfo.Name + "_" + contractAddr)
err = repository.MarkHeaderCheckedInTransaction(logs[0].Id, tx, eventId) // This assumes all logs are from same block
err = repository.MarkContractWatcherHeaderCheckedInTransaction(logs[0].Id, tx, eventId) // This assumes all logs are from same block
if err != nil {
tx.Rollback()
return err
@@ -158,7 +158,7 @@ var _ = Describe("Repository", func() {
Expect(err).ToNot(HaveOccurred())
expectedLog := test_helpers.TransferLog{
Id: 1,
VulvanizeLogId: vulcanizeLogId,
VulcanizeLogId: vulcanizeLogId,
TokenName: "TrueUSD",
Block: 5488076,
Tx: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
@@ -180,7 +180,7 @@ var _ = Describe("Repository", func() {
Expect(err).ToNot(HaveOccurred())
expectedLog := test_helpers.TransferLog{
Id: 1,
VulvanizeLogId: vulcanizeLogId,
VulcanizeLogId: vulcanizeLogId,
TokenName: "TrueUSD",
Block: 5488076,
Tx: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
+1 -1
View File
@@ -31,7 +31,7 @@ type BlockChain interface {
GetEthLogsWithCustomQuery(query ethereum.FilterQuery) ([]types.Log, error)
GetHeaderByNumber(blockNumber int64) (Header, error)
GetHeadersByNumbers(blockNumbers []int64) ([]Header, error)
GetLogs(contract Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]Log, error)
GetFullSyncLogs(contract Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]FullSyncLog, error)
GetTransactions(transactionHashes []common.Hash) ([]TransactionModel, error)
LastBlock() (*big.Int, error)
Node() Node
+1 -1
View File
@@ -18,7 +18,7 @@ package core
import "github.com/ethereum/go-ethereum/core/types"
type Log struct {
type FullSyncLog struct {
BlockNumber int64
TxHash string
Address string
+1 -1
View File
@@ -21,7 +21,7 @@ type Receipt struct {
ContractAddress string `db:"contract_address"`
CumulativeGasUsed uint64 `db:"cumulative_gas_used"`
GasUsed uint64 `db:"gas_used"`
Logs []Log
Logs []FullSyncLog
StateRoot string `db:"state_root"`
Status int
TxHash string `db:"tx_hash"`
+3 -3
View File
@@ -126,16 +126,16 @@ var _ = Describe("Postgres DB", func() {
It("does not commit log if log is invalid", func() {
//badTxHash violates db tx_hash field length
badTxHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
badLog := core.Log{
badLog := core.FullSyncLog{
Address: "x123",
BlockNumber: 1,
TxHash: badTxHash,
}
node := core.Node{GenesisBlock: "GENESIS", NetworkID: 1, ID: "x123", ClientName: "geth"}
db, _ := postgres.NewDB(test_config.DBConfig, node)
logRepository := repositories.LogRepository{DB: db}
logRepository := repositories.FullSyncLogRepository{DB: db}
err := logRepository.CreateLogs([]core.Log{badLog}, 123)
err := logRepository.CreateLogs([]core.FullSyncLog{badLog}, 123)
Expect(err).ToNot(BeNil())
savedBlock, err := logRepository.GetLogs("x123", 1)
@@ -19,10 +19,8 @@ package repositories
import (
"database/sql"
"errors"
"github.com/jmoiron/sqlx"
log "github.com/sirupsen/logrus"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -81,7 +79,7 @@ func (blockRepository BlockRepository) MissingBlockNumbers(startingBlockNumber i
startingBlockNumber,
highestBlockNumber, nodeId)
if err != nil {
log.Error("MissingBlockNumbers: error getting blocks: ", err)
logrus.Error("MissingBlockNumbers: error getting blocks: ", err)
}
return numbers
}
@@ -112,7 +110,7 @@ func (blockRepository BlockRepository) GetBlock(blockNumber int64) (core.Block,
case sql.ErrNoRows:
return core.Block{}, datastore.ErrBlockDoesNotExist(blockNumber)
default:
log.Error("GetBlock: error loading blocks: ", err)
logrus.Error("GetBlock: error loading blocks: ", err)
return savedBlock, err
}
}
@@ -151,7 +149,7 @@ func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, err
if insertBlockErr != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback transaction: ", rollbackErr)
logrus.Error("failed to rollback transaction: ", rollbackErr)
}
return 0, postgres.ErrDBInsertFailed(insertBlockErr)
}
@@ -167,7 +165,7 @@ func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, err
if insertTxErr != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Warn("failed to rollback transaction: ", rollbackErr)
logrus.Warn("failed to rollback transaction: ", rollbackErr)
}
return 0, postgres.ErrDBInsertFailed(insertTxErr)
}
@@ -176,7 +174,7 @@ func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, err
if commitErr != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Warn("failed to rollback transaction: ", rollbackErr)
logrus.Warn("failed to rollback transaction: ", rollbackErr)
}
return 0, commitErr
}
@@ -268,7 +266,7 @@ func (blockRepository BlockRepository) getBlockHash(block core.Block) (string, b
return retrievedBlockHash, blockExists(retrievedBlockHash)
}
func (blockRepository BlockRepository) createLogs(tx *sqlx.Tx, logs []core.Log, receiptId int64) error {
func (blockRepository BlockRepository) createLogs(tx *sqlx.Tx, logs []core.FullSyncLog, receiptId int64) error {
for _, tlog := range logs {
_, err := tx.Exec(
`INSERT INTO full_sync_logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)
@@ -305,7 +303,7 @@ func (blockRepository BlockRepository) loadBlock(blockRows *sqlx.Row) (core.Bloc
var block b
err := blockRows.StructScan(&block)
if err != nil {
log.Error("loadBlock: error loading block: ", err)
logrus.Error("loadBlock: error loading block: ", err)
return core.Block{}, err
}
transactionRows, err := blockRepository.database.Queryx(`
@@ -323,7 +321,7 @@ func (blockRepository BlockRepository) loadBlock(blockRows *sqlx.Row) (core.Bloc
WHERE block_id = $1
ORDER BY hash`, block.ID)
if err != nil {
log.Error("loadBlock: error fetting transactions: ", err)
logrus.Error("loadBlock: error fetting transactions: ", err)
return core.Block{}, err
}
block.Transactions = blockRepository.LoadTransactions(transactionRows)
@@ -336,7 +334,7 @@ func (blockRepository BlockRepository) LoadTransactions(transactionRows *sqlx.Ro
var transaction core.TransactionModel
err := transactionRows.StructScan(&transaction)
if err != nil {
log.Fatal(err)
logrus.Fatal(err)
}
transactions = append(transactions, transaction)
}
@@ -0,0 +1,70 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repositories
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
const (
insertCheckedHeaderQuery = `INSERT INTO public.checked_headers (header_id) VALUES ($1)
ON CONFLICT (header_id) DO UPDATE
SET check_count = (SELECT check_count FROM public.checked_headers WHERE header_id = $1) + 1`
)
type CheckedHeadersRepository struct {
db *postgres.DB
}
func NewCheckedHeadersRepository(db *postgres.DB) CheckedHeadersRepository {
return CheckedHeadersRepository{db: db}
}
// Adds header_id to the checked_headers table, or increment check_count if header_id already present
func (repo CheckedHeadersRepository) MarkHeaderChecked(headerID int64) error {
_, err := repo.db.Exec(insertCheckedHeaderQuery, headerID)
return err
}
// Return header_id if not present in checked_headers or its check_count is < passed checkCount
func (repo CheckedHeadersRepository) MissingHeaders(startingBlockNumber, endingBlockNumber, checkCount int64) ([]core.Header, error) {
var result []core.Header
var query string
var err error
if endingBlockNumber == -1 {
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR check_count < $2)
AND headers.block_number >= $1
AND headers.eth_node_fingerprint = $3
LIMIT 100`
err = repo.db.Select(&result, query, startingBlockNumber, checkCount, repo.db.Node.ID)
} else {
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR check_count < $3)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $4
LIMIT 100`
err = repo.db.Select(&result, query, startingBlockNumber, endingBlockNumber, checkCount, repo.db.Node.ID)
}
return result, err
}
@@ -0,0 +1,270 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repositories_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/test_config"
"math/rand"
)
var _ = Describe("Checked Headers repository", func() {
var (
db *postgres.DB
repo datastore.CheckedHeadersRepository
)
Describe("MarkHeaderChecked", func() {
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
repo = repositories.NewCheckedHeadersRepository(db)
})
It("marks passed header as checked on insert", func() {
headerRepository := repositories.NewHeaderRepository(db)
headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(headerErr).NotTo(HaveOccurred())
err := repo.MarkHeaderChecked(headerID)
Expect(err).NotTo(HaveOccurred())
var checkedCount int
fetchErr := db.Get(&checkedCount, `SELECT check_count FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(fetchErr).NotTo(HaveOccurred())
Expect(checkedCount).To(Equal(1))
})
It("increments check count on update", func() {
headerRepository := repositories.NewHeaderRepository(db)
headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(headerErr).NotTo(HaveOccurred())
insertErr := repo.MarkHeaderChecked(headerID)
Expect(insertErr).NotTo(HaveOccurred())
updateErr := repo.MarkHeaderChecked(headerID)
Expect(updateErr).NotTo(HaveOccurred())
var checkedCount int
fetchErr := db.Get(&checkedCount, `SELECT check_count FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(fetchErr).NotTo(HaveOccurred())
Expect(checkedCount).To(Equal(2))
})
})
Describe("MissingHeaders", func() {
var (
headerRepository datastore.HeaderRepository
startingBlockNumber int64
endingBlockNumber int64
middleBlockNumber int64
outOfRangeBlockNumber int64
blockNumbers []int64
headerIDs []int64
err error
uncheckedCheckCount = int64(1)
recheckCheckCount = int64(2)
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
headerRepository = repositories.NewHeaderRepository(db)
repo = repositories.NewCheckedHeadersRepository(db)
startingBlockNumber = rand.Int63()
middleBlockNumber = startingBlockNumber + 1
endingBlockNumber = startingBlockNumber + 2
outOfRangeBlockNumber = endingBlockNumber + 1
blockNumbers = []int64{startingBlockNumber, middleBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
headerIDs = []int64{}
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
headerIDs = append(headerIDs, headerID)
Expect(err).NotTo(HaveOccurred())
}
})
Describe("when ending block is specified", func() {
It("excludes headers that are out of range", func() {
headers, err := repo.MissingHeaders(startingBlockNumber, endingBlockNumber, uncheckedCheckCount)
Expect(err).NotTo(HaveOccurred())
// doesn't include outOfRangeBlockNumber
Expect(len(headers)).To(Equal(3))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(middleBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(middleBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(middleBlockNumber)))
})
It("excludes headers that have been checked more than the check count", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := repo.MissingHeaders(startingBlockNumber, endingBlockNumber, uncheckedCheckCount)
Expect(err).NotTo(HaveOccurred())
// doesn't include middleBlockNumber
Expect(len(headers)).To(Equal(2))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
})
It("does not exclude headers that have been checked less than the check count", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := repo.MissingHeaders(startingBlockNumber, endingBlockNumber, recheckCheckCount)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber)))
})
It("only returns headers associated with the current node", func() {
dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
repoTwo := repositories.NewCheckedHeadersRepository(dbTwo)
for _, n := range blockNumbers {
_, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10))
Expect(err).NotTo(HaveOccurred())
}
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := repo.MissingHeaders(startingBlockNumber, endingBlockNumber, uncheckedCheckCount)
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeOneMissingHeaders)).To(Equal(3))
Expect(nodeOneMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber)))
Expect(nodeOneMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber)))
Expect(nodeOneMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber)))
nodeTwoMissingHeaders, err := repoTwo.MissingHeaders(startingBlockNumber, endingBlockNumber+10, uncheckedCheckCount)
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeTwoMissingHeaders)).To(Equal(3))
Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(middleBlockNumber+10), Equal(endingBlockNumber+10)))
Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(middleBlockNumber+10), Equal(endingBlockNumber+10)))
Expect(nodeTwoMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(middleBlockNumber+10), Equal(endingBlockNumber+10)))
})
It("only returns 100 results to prevent blocking log delegation", func() {
for n := outOfRangeBlockNumber + 1; n < outOfRangeBlockNumber+100; n++ {
_, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
Expect(err).NotTo(HaveOccurred())
}
missingHeaders, err := repo.MissingHeaders(startingBlockNumber, endingBlockNumber+200, uncheckedCheckCount)
Expect(err).NotTo(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(100))
})
})
Describe("when ending block is -1", func() {
var endingBlock = int64(-1)
It("includes all non-checked headers when ending block is -1 ", func() {
headers, err := repo.MissingHeaders(startingBlockNumber, endingBlock, uncheckedCheckCount)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(4))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(middleBlockNumber), Equal(outOfRangeBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(middleBlockNumber), Equal(outOfRangeBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(middleBlockNumber), Equal(outOfRangeBlockNumber)))
Expect(headers[3].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(middleBlockNumber), Equal(outOfRangeBlockNumber)))
})
It("excludes headers that have been checked more than the check count", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := repo.MissingHeaders(startingBlockNumber, endingBlock, uncheckedCheckCount)
Expect(err).NotTo(HaveOccurred())
// doesn't include middleBlockNumber
Expect(len(headers)).To(Equal(3))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
})
It("does not exclude headers that have been checked less than the check count", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
headers, err := repo.MissingHeaders(startingBlockNumber, endingBlock, recheckCheckCount)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(4))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
Expect(headers[3].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
})
It("only returns headers associated with the current node", func() {
dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
repoTwo := repositories.NewCheckedHeadersRepository(dbTwo)
for _, n := range blockNumbers {
_, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10))
Expect(err).NotTo(HaveOccurred())
}
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := repo.MissingHeaders(startingBlockNumber, endingBlock, uncheckedCheckCount)
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeOneMissingHeaders)).To(Equal(4))
Expect(nodeOneMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
Expect(nodeOneMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
Expect(nodeOneMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
Expect(nodeOneMissingHeaders[3].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
nodeTwoMissingHeaders, err := repoTwo.MissingHeaders(startingBlockNumber, endingBlock, uncheckedCheckCount)
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeTwoMissingHeaders)).To(Equal(4))
Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(middleBlockNumber+10), Equal(endingBlockNumber+10), Equal(outOfRangeBlockNumber+10)))
Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(middleBlockNumber+10), Equal(endingBlockNumber+10), Equal(outOfRangeBlockNumber+10)))
Expect(nodeTwoMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(middleBlockNumber+10), Equal(endingBlockNumber+10), Equal(outOfRangeBlockNumber+10)))
Expect(nodeTwoMissingHeaders[3].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(middleBlockNumber+10), Equal(endingBlockNumber+10), Equal(outOfRangeBlockNumber+10)))
})
It("only returns 100 results to prevent blocking log delegation", func() {
for n := outOfRangeBlockNumber + 1; n < outOfRangeBlockNumber+100; n++ {
_, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
Expect(err).NotTo(HaveOccurred())
}
missingHeaders, err := repo.MissingHeaders(startingBlockNumber, endingBlock, uncheckedCheckCount)
Expect(err).NotTo(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(100))
})
})
})
})
@@ -17,20 +17,18 @@
package repositories
import (
"github.com/sirupsen/logrus"
"database/sql"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type LogRepository struct {
type FullSyncLogRepository struct {
*postgres.DB
}
func (logRepository LogRepository) CreateLogs(lgs []core.Log, receiptId int64) error {
tx, _ := logRepository.DB.Beginx()
func (repository FullSyncLogRepository) CreateLogs(lgs []core.FullSyncLog, receiptId int64) error {
tx, _ := repository.DB.Beginx()
for _, tlog := range lgs {
_, insertLogErr := tx.Exec(
`INSERT INTO full_sync_logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)
@@ -57,8 +55,8 @@ func (logRepository LogRepository) CreateLogs(lgs []core.Log, receiptId int64) e
return nil
}
func (logRepository LogRepository) GetLogs(address string, blockNumber int64) ([]core.Log, error) {
logRows, err := logRepository.DB.Query(
func (repository FullSyncLogRepository) GetLogs(address string, blockNumber int64) ([]core.FullSyncLog, error) {
logRows, err := repository.DB.Query(
`SELECT block_number,
address,
tx_hash,
@@ -72,13 +70,13 @@ func (logRepository LogRepository) GetLogs(address string, blockNumber int64) ([
WHERE address = $1 AND block_number = $2
ORDER BY block_number DESC`, address, blockNumber)
if err != nil {
return []core.Log{}, err
return []core.FullSyncLog{}, err
}
return logRepository.loadLogs(logRows)
return repository.loadLogs(logRows)
}
func (logRepository LogRepository) loadLogs(logsRows *sql.Rows) ([]core.Log, error) {
var lgs []core.Log
func (repository FullSyncLogRepository) loadLogs(logsRows *sql.Rows) ([]core.FullSyncLog, error) {
var lgs []core.FullSyncLog
for logsRows.Next() {
var blockNumber int64
var address string
@@ -89,9 +87,9 @@ func (logRepository LogRepository) loadLogs(logsRows *sql.Rows) ([]core.Log, err
err := logsRows.Scan(&blockNumber, &address, &txHash, &index, &topics[0], &topics[1], &topics[2], &topics[3], &data)
if err != nil {
logrus.Error("loadLogs: Error scanning a row in logRows: ", err)
return []core.Log{}, err
return []core.FullSyncLog{}, err
}
lg := core.Log{
lg := core.FullSyncLog{
BlockNumber: blockNumber,
TxHash: txHash,
Address: address,
@@ -29,11 +29,11 @@ import (
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Logs Repository", func() {
var _ = Describe("Full sync log Repository", func() {
Describe("Saving logs", func() {
var db *postgres.DB
var blockRepository datastore.BlockRepository
var logsRepository datastore.LogRepository
var logsRepository datastore.FullSyncLogRepository
var receiptRepository datastore.FullSyncReceiptRepository
var node core.Node
@@ -47,7 +47,7 @@ var _ = Describe("Logs Repository", func() {
db = test_config.NewTestDB(node)
test_config.CleanTestDB(db)
blockRepository = repositories.NewBlockRepository(db)
logsRepository = repositories.LogRepository{DB: db}
logsRepository = repositories.FullSyncLogRepository{DB: db}
receiptRepository = repositories.FullSyncReceiptRepository{DB: db}
})
@@ -59,7 +59,7 @@ var _ = Describe("Logs Repository", func() {
receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx)
tx.Commit()
Expect(err).NotTo(HaveOccurred())
err = logsRepository.CreateLogs([]core.Log{{
err = logsRepository.CreateLogs([]core.FullSyncLog{{
BlockNumber: blockNumber,
Index: 0,
Address: "x123",
@@ -98,7 +98,7 @@ var _ = Describe("Logs Repository", func() {
tx.Commit()
Expect(err).NotTo(HaveOccurred())
err = logsRepository.CreateLogs([]core.Log{{
err = logsRepository.CreateLogs([]core.FullSyncLog{{
BlockNumber: blockNumber,
Index: 0,
Address: "x123",
@@ -108,7 +108,7 @@ var _ = Describe("Logs Repository", func() {
}}, receiptId)
Expect(err).NotTo(HaveOccurred())
err = logsRepository.CreateLogs([]core.Log{{
err = logsRepository.CreateLogs([]core.FullSyncLog{{
BlockNumber: blockNumber,
Index: 1,
Address: "x123",
@@ -118,7 +118,7 @@ var _ = Describe("Logs Repository", func() {
}}, receiptId)
Expect(err).NotTo(HaveOccurred())
err = logsRepository.CreateLogs([]core.Log{{
err = logsRepository.CreateLogs([]core.FullSyncLog{{
BlockNumber: 2,
Index: 0,
Address: "x123",
@@ -162,7 +162,7 @@ var _ = Describe("Logs Repository", func() {
It("saves the logs attached to a receipt", func() {
logs := []core.Log{{
logs := []core.FullSyncLog{{
Address: "0x8a4774fe82c63484afef97ca8d89a6ea5e21f973",
BlockNumber: 4745407,
Data: "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000645a68669900000000000000000000000000000000000000000000003397684ab5869b0000000000000000000000000000000000000000000000000000000000005a36053200000000000000000000000099041f808d598b782d5a3e498681c2452a31da08",
@@ -20,7 +20,6 @@ import (
"database/sql"
"github.com/jmoiron/sqlx"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -53,7 +52,22 @@ func (receiptRepository FullSyncReceiptRepository) CreateReceiptsAndLogs(blockId
return nil
}
func createLogs(logs []core.Log, receiptId int64, tx *sqlx.Tx) error {
func createReceipt(receipt core.Receipt, blockId int64, tx *sqlx.Tx) (int64, error) {
var receiptId int64
err := tx.QueryRow(
`INSERT INTO full_sync_receipts
(contract_address, tx_hash, cumulative_gas_used, gas_used, state_root, status, block_id)
VALUES ($1, $2, $3, $4, $5, $6, $7)
RETURNING id`,
receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, blockId,
).Scan(&receiptId)
if err != nil {
logrus.Error("createReceipt: Error inserting: ", err)
}
return receiptId, err
}
func createLogs(logs []core.FullSyncLog, receiptId int64, tx *sqlx.Tx) error {
for _, log := range logs {
_, err := tx.Exec(
`INSERT INTO full_sync_logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)
@@ -29,7 +29,7 @@ import (
var _ = Describe("Receipt Repository", func() {
var blockRepository datastore.BlockRepository
var logRepository datastore.LogRepository
var logRepository datastore.FullSyncLogRepository
var receiptRepository datastore.FullSyncReceiptRepository
var db *postgres.DB
var node core.Node
@@ -43,7 +43,7 @@ var _ = Describe("Receipt Repository", func() {
db = test_config.NewTestDB(node)
test_config.CleanTestDB(db)
blockRepository = repositories.NewBlockRepository(db)
logRepository = repositories.LogRepository{DB: db}
logRepository = repositories.FullSyncLogRepository{DB: db}
receiptRepository = repositories.FullSyncReceiptRepository{DB: db}
})
@@ -56,7 +56,7 @@ var _ = Describe("Receipt Repository", func() {
txHashTwo := "0xTxHashTwo"
addressOne := "0xAddressOne"
addressTwo := "0xAddressTwo"
logsOne := []core.Log{{
logsOne := []core.FullSyncLog{{
Address: addressOne,
BlockNumber: blockNumber,
TxHash: txHashOne,
@@ -65,7 +65,7 @@ var _ = Describe("Receipt Repository", func() {
BlockNumber: blockNumber,
TxHash: txHashOne,
}}
logsTwo := []core.Log{{
logsTwo := []core.FullSyncLog{{
BlockNumber: blockNumber,
TxHash: txHashTwo,
Address: addressTwo,
@@ -112,7 +112,7 @@ var _ = Describe("Receipt Repository", func() {
ContractAddress: "0xde0b295669a9fd93d5f28d9ec85e40f4cb697bae",
CumulativeGasUsed: 7996119,
GasUsed: 21000,
Logs: []core.Log{},
Logs: []core.FullSyncLog{},
StateRoot: "0x88abf7e73128227370aa7baa3dd4e18d0af70e92ef1f9ef426942fbe2dddb733",
Status: 1,
TxHash: "0xe340558980f89d5f86045ac11e5cc34e4bcec20f9f1e2a427aa39d87114e8223",
@@ -0,0 +1,134 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repositories
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
const insertHeaderSyncLogQuery = `INSERT INTO header_sync_logs
(header_id, address, topics, data, block_number, block_hash, tx_index, tx_hash, log_index, raw)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT DO NOTHING`
type HeaderSyncLogRepository struct {
db *postgres.DB
}
func NewHeaderSyncLogRepository(db *postgres.DB) HeaderSyncLogRepository {
return HeaderSyncLogRepository{db: db}
}
type headerSyncLog struct {
ID int64
HeaderID int64 `db:"header_id"`
Address string
Topics pq.ByteaArray
Data []byte
BlockNumber uint64 `db:"block_number"`
BlockHash string `db:"block_hash"`
TxHash string `db:"tx_hash"`
TxIndex uint `db:"tx_index"`
LogIndex uint `db:"log_index"`
Transformed bool
Raw []byte
}
func (repository HeaderSyncLogRepository) GetUntransformedHeaderSyncLogs() ([]core.HeaderSyncLog, error) {
rows, queryErr := repository.db.Queryx(`SELECT * FROM public.header_sync_logs WHERE transformed = false`)
if queryErr != nil {
return nil, queryErr
}
var results []core.HeaderSyncLog
for rows.Next() {
var rawLog headerSyncLog
scanErr := rows.StructScan(&rawLog)
if scanErr != nil {
return nil, scanErr
}
var logTopics []common.Hash
for _, topic := range rawLog.Topics {
logTopics = append(logTopics, common.BytesToHash(topic))
}
reconstructedLog := types.Log{
Address: common.HexToAddress(rawLog.Address),
Topics: logTopics,
Data: rawLog.Data,
BlockNumber: rawLog.BlockNumber,
TxHash: common.HexToHash(rawLog.TxHash),
TxIndex: rawLog.TxIndex,
BlockHash: common.HexToHash(rawLog.BlockHash),
Index: rawLog.LogIndex,
// TODO: revisit if not cascade deleting logs when header removed
// currently, fetched logs are cascade deleted if removed
Removed: false,
}
result := core.HeaderSyncLog{
ID: rawLog.ID,
HeaderID: rawLog.HeaderID,
Log: reconstructedLog,
Transformed: rawLog.Transformed,
}
// TODO: Consider returning each result async to avoid keeping large result sets in memory
results = append(results, result)
}
return results, nil
}
func (repository HeaderSyncLogRepository) CreateHeaderSyncLogs(headerID int64, logs []types.Log) error {
tx, txErr := repository.db.Beginx()
if txErr != nil {
return txErr
}
for _, log := range logs {
err := insertLog(headerID, log, tx)
if err != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
logrus.Errorf("failed to rollback header sync log insert: %s", rollbackErr.Error())
}
return err
}
}
return tx.Commit()
}
func insertLog(headerID int64, log types.Log, tx *sqlx.Tx) error {
topics := buildTopics(log)
raw, jsonErr := log.MarshalJSON()
if jsonErr != nil {
return jsonErr
}
_, insertErr := tx.Exec(insertHeaderSyncLogQuery, headerID, log.Address.Hex(), topics, log.Data, log.BlockNumber,
log.BlockHash.Hex(), log.TxIndex, log.TxHash.Hex(), log.Index, raw)
return insertErr
}
func buildTopics(log types.Log) pq.ByteaArray {
var topics pq.ByteaArray
for _, topic := range log.Topics {
topics = append(topics, topic.Bytes())
}
return topics
}
@@ -0,0 +1,203 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repositories_test
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/lib/pq"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Header sync log repository", func() {
var (
db *postgres.DB
headerID int64
repository datastore.HeaderSyncLogRepository
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
var headerErr error
headerID, headerErr = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(headerErr).NotTo(HaveOccurred())
repository = repositories.NewHeaderSyncLogRepository(db)
})
Describe("CreateHeaderSyncLogs", func() {
type HeaderSyncLog struct {
ID int64
HeaderID int64 `db:"header_id"`
Address string
Topics pq.ByteaArray
Data []byte
BlockNumber uint64 `db:"block_number"`
BlockHash string `db:"block_hash"`
TxHash string `db:"tx_hash"`
TxIndex uint `db:"tx_index"`
LogIndex uint `db:"log_index"`
Transformed bool
Raw []byte
}
It("writes a log to the db", func() {
log := test_data.GenericTestLog()
err := repository.CreateHeaderSyncLogs(headerID, []types.Log{log})
Expect(err).NotTo(HaveOccurred())
var dbLog HeaderSyncLog
lookupErr := db.Get(&dbLog, `SELECT * FROM header_sync_logs`)
Expect(lookupErr).NotTo(HaveOccurred())
Expect(dbLog.ID).NotTo(BeZero())
Expect(dbLog.HeaderID).To(Equal(headerID))
Expect(dbLog.Address).To(Equal(log.Address.Hex()))
Expect(dbLog.Topics[0]).To(Equal(log.Topics[0].Bytes()))
Expect(dbLog.Topics[1]).To(Equal(log.Topics[1].Bytes()))
Expect(dbLog.Data).To(Equal(log.Data))
Expect(dbLog.BlockNumber).To(Equal(log.BlockNumber))
Expect(dbLog.BlockHash).To(Equal(log.BlockHash.Hex()))
Expect(dbLog.TxIndex).To(Equal(log.TxIndex))
Expect(dbLog.TxHash).To(Equal(log.TxHash.Hex()))
Expect(dbLog.LogIndex).To(Equal(log.Index))
expectedRaw, jsonErr := log.MarshalJSON()
Expect(jsonErr).NotTo(HaveOccurred())
Expect(dbLog.Raw).To(MatchJSON(expectedRaw))
Expect(dbLog.Transformed).To(BeFalse())
})
It("writes several logs to the db", func() {
log1 := test_data.GenericTestLog()
log2 := test_data.GenericTestLog()
logs := []types.Log{log1, log2}
err := repository.CreateHeaderSyncLogs(headerID, logs)
Expect(err).NotTo(HaveOccurred())
var count int
lookupErr := db.Get(&count, `SELECT COUNT(*) FROM header_sync_logs`)
Expect(lookupErr).NotTo(HaveOccurred())
Expect(count).To(Equal(len(logs)))
})
It("persists record that can be unpacked into types.Log", func() {
// important if we want to decouple log persistence from transforming and still make use of
// tools on types.Log like abi.Unpack
log := test_data.GenericTestLog()
err := repository.CreateHeaderSyncLogs(headerID, []types.Log{log})
Expect(err).NotTo(HaveOccurred())
var dbLog HeaderSyncLog
lookupErr := db.Get(&dbLog, `SELECT * FROM header_sync_logs`)
Expect(lookupErr).NotTo(HaveOccurred())
var logTopics []common.Hash
for _, topic := range dbLog.Topics {
logTopics = append(logTopics, common.BytesToHash(topic))
}
reconstructedLog := types.Log{
Address: common.HexToAddress(dbLog.Address),
Topics: logTopics,
Data: dbLog.Data,
BlockNumber: dbLog.BlockNumber,
TxHash: common.HexToHash(dbLog.TxHash),
TxIndex: dbLog.TxIndex,
BlockHash: common.HexToHash(dbLog.BlockHash),
Index: dbLog.LogIndex,
Removed: false,
}
Expect(reconstructedLog).To(Equal(log))
})
It("does not duplicate logs", func() {
log := test_data.GenericTestLog()
err := repository.CreateHeaderSyncLogs(headerID, []types.Log{log, log})
Expect(err).NotTo(HaveOccurred())
var count int
lookupErr := db.Get(&count, `SELECT COUNT(*) FROM header_sync_logs`)
Expect(lookupErr).NotTo(HaveOccurred())
Expect(count).To(Equal(1))
})
})
Describe("GetFullSyncLogs", func() {
Describe("when there are no logs", func() {
It("returns empty collection", func() {
result, err := repository.GetUntransformedHeaderSyncLogs()
Expect(err).NotTo(HaveOccurred())
Expect(len(result)).To(BeZero())
})
})
Describe("when there are logs", func() {
var log1, log2 types.Log
BeforeEach(func() {
log1 = test_data.GenericTestLog()
log2 = test_data.GenericTestLog()
logs := []types.Log{log1, log2}
logsErr := repository.CreateHeaderSyncLogs(headerID, logs)
Expect(logsErr).NotTo(HaveOccurred())
})
It("returns persisted logs", func() {
result, err := repository.GetUntransformedHeaderSyncLogs()
Expect(err).NotTo(HaveOccurred())
Expect(len(result)).To(Equal(2))
Expect(result[0].Log).To(Or(Equal(log1), Equal(log2)))
Expect(result[1].Log).To(Or(Equal(log1), Equal(log2)))
Expect(result[0].Log).NotTo(Equal(result[1].Log))
})
It("excludes logs that have been transformed", func() {
_, insertErr := db.Exec(`UPDATE public.header_sync_logs SET transformed = true WHERE tx_hash = $1`, log1.TxHash.Hex())
Expect(insertErr).NotTo(HaveOccurred())
result, err := repository.GetUntransformedHeaderSyncLogs()
Expect(err).NotTo(HaveOccurred())
Expect(len(result)).To(Equal(1))
Expect(result[0].Log).To(Equal(log2))
})
It("returns empty collection if all logs transformed", func() {
_, insertErr := db.Exec(`UPDATE public.header_sync_logs SET transformed = true WHERE header_id = $1`, headerID)
Expect(insertErr).NotTo(HaveOccurred())
result, err := repository.GetUntransformedHeaderSyncLogs()
Expect(err).NotTo(HaveOccurred())
Expect(len(result)).To(BeZero())
})
})
})
})
@@ -32,7 +32,7 @@ var _ = Describe("Watched Events Repository", func() {
var db *postgres.DB
var blocksRepository datastore.BlockRepository
var filterRepository datastore.FilterRepository
var logRepository datastore.LogRepository
var logRepository datastore.FullSyncLogRepository
var receiptRepository datastore.FullSyncReceiptRepository
var watchedEventRepository datastore.WatchedEventRepository
@@ -41,7 +41,7 @@ var _ = Describe("Watched Events Repository", func() {
test_config.CleanTestDB(db)
blocksRepository = repositories.NewBlockRepository(db)
filterRepository = repositories.FilterRepository{DB: db}
logRepository = repositories.LogRepository{DB: db}
logRepository = repositories.FullSyncLogRepository{DB: db}
receiptRepository = repositories.FullSyncReceiptRepository{DB: db}
watchedEventRepository = repositories.WatchedEventRepository{DB: db}
})
@@ -54,7 +54,7 @@ var _ = Describe("Watched Events Repository", func() {
Address: "0x123",
Topics: core.Topics{0: "event1=10", 2: "event3=hello"},
}
logs := []core.Log{
logs := []core.FullSyncLog{
{
BlockNumber: 0,
TxHash: "0x1",
@@ -108,7 +108,7 @@ var _ = Describe("Watched Events Repository", func() {
Address: "0x123",
Topics: core.Topics{0: "event1=10", 2: "event3=hello"},
}
logs := []core.Log{
logs := []core.FullSyncLog{
{
BlockNumber: 0,
TxHash: "0x1",
+14 -3
View File
@@ -17,6 +17,7 @@
package datastore
import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/jmoiron/sqlx"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/filters"
@@ -33,6 +34,11 @@ type BlockRepository interface {
SetBlocksStatus(chainHead int64) error
}
type CheckedHeadersRepository interface {
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber, checkCount int64) ([]core.Header, error)
}
type ContractRepository interface {
CreateContract(contract core.Contract) error
GetContract(contractHash string) (core.Contract, error)
@@ -44,6 +50,11 @@ type FilterRepository interface {
GetFilter(name string) (filters.LogFilter, error)
}
type FullSyncLogRepository interface {
CreateLogs(logs []core.FullSyncLog, receiptId int64) error
GetLogs(address string, blockNumber int64) ([]core.FullSyncLog, error)
}
type HeaderRepository interface {
CreateOrUpdateHeader(header core.Header) (int64, error)
CreateTransactions(headerID int64, transactions []core.TransactionModel) error
@@ -51,9 +62,9 @@ type HeaderRepository interface {
MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) ([]int64, error)
}
type LogRepository interface {
CreateLogs(logs []core.Log, receiptId int64) error
GetLogs(address string, blockNumber int64) ([]core.Log, error)
type HeaderSyncLogRepository interface {
GetUntransformedHeaderSyncLogs() ([]core.HeaderSyncLog, error)
CreateHeaderSyncLogs(headerID int64, logs []types.Log) error
}
type FullSyncReceiptRepository interface {
+2 -2
View File
@@ -107,8 +107,8 @@ func (chain *MockBlockChain) GetHeadersByNumbers(blockNumbers []int64) ([]core.H
return headers, nil
}
func (chain *MockBlockChain) GetLogs(contract core.Contract, startingBlockNumber, endingBlockNumber *big.Int) ([]core.Log, error) {
return []core.Log{}, nil
func (chain *MockBlockChain) GetFullSyncLogs(contract core.Contract, startingBlockNumber, endingBlockNumber *big.Int) ([]core.FullSyncLog, error) {
return []core.FullSyncLog{}, nil
}
func (chain *MockBlockChain) GetTransactions(transactionHashes []common.Hash) ([]core.TransactionModel, error) {
@@ -0,0 +1,43 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package fakes
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
)
type MockCheckedHeadersRepository struct {
CheckCount int64
StartingBlockNumber int64
EndingBlockNumber int64
HeaderID int64
ReturnHeaders []core.Header
MarkHeaderCheckedReturnError error
MissingHeadersReturnError error
}
func (repository *MockCheckedHeadersRepository) MarkHeaderChecked(headerID int64) error {
repository.HeaderID = headerID
return repository.MarkHeaderCheckedReturnError
}
func (repository *MockCheckedHeadersRepository) MissingHeaders(startingBlockNumber, endingBlockNumber, checkCount int64) ([]core.Header, error) {
repository.StartingBlockNumber = startingBlockNumber
repository.EndingBlockNumber = endingBlockNumber
repository.CheckCount = checkCount
return repository.ReturnHeaders, repository.MissingHeadersReturnError
}
@@ -0,0 +1,42 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package fakes
import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/core"
)
type MockHeaderSyncLogRepository struct {
CreateError error
GetCalled bool
GetError error
PassedHeaderID int64
PassedLogs []types.Log
ReturnLogs []core.HeaderSyncLog
}
func (repository *MockHeaderSyncLogRepository) GetUntransformedHeaderSyncLogs() ([]core.HeaderSyncLog, error) {
repository.GetCalled = true
return repository.ReturnLogs, repository.GetError
}
func (repository *MockHeaderSyncLogRepository) CreateHeaderSyncLogs(headerID int64, logs []types.Log) error {
repository.PassedHeaderID = headerID
repository.PassedLogs = logs
return repository.CreateError
}
+3 -3
View File
@@ -86,7 +86,7 @@ func (blockChain *BlockChain) GetHeadersByNumbers(blockNumbers []int64) (header
return blockChain.getPOWHeaders(blockNumbers)
}
func (blockChain *BlockChain) GetLogs(contract core.Contract, startingBlockNumber, endingBlockNumber *big.Int) ([]core.Log, error) {
func (blockChain *BlockChain) GetFullSyncLogs(contract core.Contract, startingBlockNumber, endingBlockNumber *big.Int) ([]core.FullSyncLog, error) {
if endingBlockNumber == nil {
endingBlockNumber = startingBlockNumber
}
@@ -99,9 +99,9 @@ func (blockChain *BlockChain) GetLogs(contract core.Contract, startingBlockNumbe
}
gethLogs, err := blockChain.GetEthLogsWithCustomQuery(fc)
if err != nil {
return []core.Log{}, err
return []core.FullSyncLog{}, err
}
logs := vulcCommon.ToCoreLogs(gethLogs)
logs := vulcCommon.ToFullSyncLogs(gethLogs)
return logs, nil
}
+2 -2
View File
@@ -154,7 +154,7 @@ var _ = Describe("Geth blockchain", func() {
startingBlockNumber := big.NewInt(1)
endingBlockNumber := big.NewInt(2)
_, err := blockChain.GetLogs(contract, startingBlockNumber, endingBlockNumber)
_, err := blockChain.GetFullSyncLogs(contract, startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
expectedQuery := ethereum.FilterQuery{
@@ -171,7 +171,7 @@ var _ = Describe("Geth blockchain", func() {
startingBlockNumber := big.NewInt(1)
endingBlockNumber := big.NewInt(2)
_, err := blockChain.GetLogs(contract, startingBlockNumber, endingBlockNumber)
_, err := blockChain.GetFullSyncLogs(contract, startingBlockNumber, endingBlockNumber)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
@@ -26,8 +26,8 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core"
)
func ToCoreLogs(gethLogs []types.Log) []core.Log {
var logs []core.Log
func ToFullSyncLogs(gethLogs []types.Log) []core.FullSyncLog {
var logs []core.FullSyncLog
for _, log := range gethLogs {
log := ToCoreLog(log)
logs = append(logs, log)
@@ -43,10 +43,10 @@ func makeTopics(topics []common.Hash) core.Topics {
return hexTopics
}
func ToCoreLog(gethLog types.Log) core.Log {
func ToCoreLog(gethLog types.Log) core.FullSyncLog {
topics := gethLog.Topics
hexTopics := makeTopics(topics)
return core.Log{
return core.FullSyncLog{
Address: strings.ToLower(gethLog.Address.Hex()),
BlockNumber: int64(gethLog.BlockNumber),
Topics: hexTopics,
@@ -29,7 +29,7 @@ import (
vulcCommon "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
)
var _ = Describe("Conversion of GethLog to core.Log", func() {
var _ = Describe("Conversion of GethLog to core.FullSyncLog", func() {
It("converts geth log to internal log format", func() {
gethLog := types.Log{
@@ -46,7 +46,7 @@ var _ = Describe("Conversion of GethLog to core.Log", func() {
},
}
expected := core.Log{
expected := core.FullSyncLog{
Address: strings.ToLower(gethLog.Address.Hex()),
BlockNumber: int64(gethLog.BlockNumber),
Data: hexutil.Encode(gethLog.Data),
@@ -101,7 +101,7 @@ var _ = Describe("Conversion of GethLog to core.Log", func() {
expectedOne := vulcCommon.ToCoreLog(gethLogOne)
expectedTwo := vulcCommon.ToCoreLog(gethLogTwo)
coreLogs := vulcCommon.ToCoreLogs([]types.Log{gethLogOne, gethLogTwo})
coreLogs := vulcCommon.ToFullSyncLogs([]types.Log{gethLogOne, gethLogTwo})
Expect(len(coreLogs)).To(Equal(2))
Expect(coreLogs[0]).To(Equal(expectedOne))
@@ -73,8 +73,8 @@ func setContractAddress(gethReceipt *types.Receipt) string {
return gethReceipt.ContractAddress.Hex()
}
func dereferenceLogs(gethReceipt *types.Receipt) []core.Log {
logs := []core.Log{}
func dereferenceLogs(gethReceipt *types.Receipt) []core.FullSyncLog {
logs := []core.FullSyncLog{}
for _, log := range gethReceipt.Logs {
logs = append(logs, ToCoreLog(*log))
}
@@ -51,7 +51,7 @@ var _ = Describe("Conversion of GethReceipt to core.Receipt", func() {
ContractAddress: "",
CumulativeGasUsed: 25000,
GasUsed: 21000,
Logs: []core.Log{},
Logs: []core.FullSyncLog{},
StateRoot: "0x88abf7e73128227370aa7baa3dd4e18d0af70e92ef1f9ef426942fbe2dddb733",
Status: -99,
TxHash: receipt.TxHash.Hex(),
@@ -92,7 +92,7 @@ var _ = Describe("Conversion of GethReceipt to core.Receipt", func() {
ContractAddress: receipt.ContractAddress.Hex(),
CumulativeGasUsed: 7996119,
GasUsed: 21000,
Logs: []core.Log{},
Logs: []core.FullSyncLog{},
StateRoot: "",
Status: 1,
TxHash: receipt.TxHash.Hex(),