Decouple log extraction from transformer delegation
- limit missing headers results set to 100 so that extraction doesn't excessively block delegation - wrap checked headers functions in repository struct - move storage repository to factory, to correspond with event repository path - remove unused files - reformat sql - remove line breaks in imports
This commit is contained in:
Rob Mulholand
@@ -16,192 +16,12 @@
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"fmt"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/lib/pq"
|
||||
"github.com/sirupsen/logrus"
|
||||
import "github.com/jmoiron/sqlx"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
)
|
||||
|
||||
const insertHeaderSyncLogQuery = `INSERT INTO header_sync_logs
|
||||
(header_id, address, topics, data, block_number, block_hash, tx_index, tx_hash, log_index, raw)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT DO NOTHING RETURNING id`
|
||||
|
||||
func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error {
|
||||
_, err := db.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
|
||||
VALUES ($1, $2)
|
||||
ON CONFLICT (header_id) DO
|
||||
UPDATE SET `+checkedHeadersColumn+` = checked_headers.`+checkedHeadersColumn+` + 1`, headerID, 1)
|
||||
return err
|
||||
}
|
||||
|
||||
func MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, checkedHeadersColumn string) error {
|
||||
func MarkContractWatcherHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, checkedHeadersColumn string) error {
|
||||
_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
|
||||
VALUES ($1, $2)
|
||||
ON CONFLICT (header_id) DO
|
||||
UPDATE SET `+checkedHeadersColumn+` = checked_headers.`+checkedHeadersColumn+` + 1`, headerID, 1)
|
||||
UPDATE SET `+checkedHeadersColumn+` = checked_headers.`+checkedHeadersColumn+` + 1`, headerID, 1)
|
||||
return err
|
||||
}
|
||||
|
||||
// Treats a header as missing if it's not in the headers table, or not checked for some log type
|
||||
func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) {
|
||||
var result []core.Header
|
||||
var query string
|
||||
var err error
|
||||
|
||||
if endingBlockNumber == -1 {
|
||||
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
|
||||
LEFT JOIN checked_headers on headers.id = header_id
|
||||
WHERE (header_id ISNULL OR ` + notCheckedSQL + `)
|
||||
AND headers.block_number >= $1
|
||||
AND headers.eth_node_fingerprint = $2`
|
||||
err = db.Select(&result, query, startingBlockNumber, db.Node.ID)
|
||||
} else {
|
||||
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
|
||||
LEFT JOIN checked_headers on headers.id = header_id
|
||||
WHERE (header_id ISNULL OR ` + notCheckedSQL + `)
|
||||
AND headers.block_number >= $1
|
||||
AND headers.block_number <= $2
|
||||
AND headers.eth_node_fingerprint = $3`
|
||||
err = db.Select(&result, query, startingBlockNumber, endingBlockNumber, db.Node.ID)
|
||||
}
|
||||
|
||||
return result, err
|
||||
}
|
||||
|
||||
func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
|
||||
// Query returns `[]driver.Value`, nullable polymorphic interface
|
||||
var queryResult []driver.Value
|
||||
columnNamesQuery :=
|
||||
`SELECT column_name FROM information_schema.columns
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'checked_headers'
|
||||
AND column_name <> 'id'
|
||||
AND column_name <> 'header_id';`
|
||||
|
||||
err := db.Select(&queryResult, columnNamesQuery)
|
||||
if err != nil {
|
||||
return []string{}, err
|
||||
}
|
||||
|
||||
// Transform column names from `driver.Value` to strings
|
||||
var columnNames []string
|
||||
for _, result := range queryResult {
|
||||
if columnName, ok := result.(string); ok {
|
||||
columnNames = append(columnNames, columnName)
|
||||
} else {
|
||||
return []string{}, fmt.Errorf("incorrect value for checked_headers column name")
|
||||
}
|
||||
}
|
||||
|
||||
return columnNames, nil
|
||||
}
|
||||
|
||||
// Builds a SQL string that checks if any column should be checked/rechecked.
|
||||
// Defaults to FALSE when no columns are provided.
|
||||
// Ex: ["columnA", "columnB"] => "NOT (columnA!=0 AND columnB!=0)"
|
||||
// [] => "FALSE"
|
||||
func CreateHeaderCheckedPredicateSQL(boolColumns []string, recheckHeaders constants.TransformerExecution) string {
|
||||
if len(boolColumns) == 0 {
|
||||
return "FALSE"
|
||||
}
|
||||
|
||||
if recheckHeaders {
|
||||
return createHeaderCheckedPredicateSQLForRecheckedHeaders(boolColumns)
|
||||
} else {
|
||||
return createHeaderCheckedPredicateSQLForMissingHeaders(boolColumns)
|
||||
}
|
||||
}
|
||||
|
||||
func CreateLogs(headerID int64, logs []types.Log, db *postgres.DB) ([]core.HeaderSyncLog, error) {
|
||||
tx, txErr := db.Beginx()
|
||||
if txErr != nil {
|
||||
return nil, txErr
|
||||
}
|
||||
var results []core.HeaderSyncLog
|
||||
for _, log := range logs {
|
||||
logID, err := insertLog(headerID, log, tx)
|
||||
if err != nil {
|
||||
if logWasADuplicate(err) {
|
||||
continue
|
||||
}
|
||||
rollbackErr := tx.Rollback()
|
||||
if rollbackErr != nil {
|
||||
logrus.Errorf("failed to rollback header sync log insert: %s", rollbackErr.Error())
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
results = append(results, buildLog(logID, headerID, log))
|
||||
}
|
||||
return results, tx.Commit()
|
||||
}
|
||||
|
||||
func logWasADuplicate(err error) bool {
|
||||
return err == sql.ErrNoRows
|
||||
}
|
||||
|
||||
func insertLog(headerID int64, log types.Log, tx *sqlx.Tx) (int64, error) {
|
||||
topics := buildTopics(log)
|
||||
raw, jsonErr := log.MarshalJSON()
|
||||
if jsonErr != nil {
|
||||
return 0, jsonErr
|
||||
}
|
||||
var logID int64
|
||||
err := tx.QueryRowx(insertHeaderSyncLogQuery, headerID, log.Address.Hex(), topics, log.Data, log.BlockNumber,
|
||||
log.BlockHash.Hex(), log.TxIndex, log.TxHash.Hex(), log.Index, raw).Scan(&logID)
|
||||
return logID, err
|
||||
}
|
||||
|
||||
func buildLog(logID int64, headerID int64, log types.Log) core.HeaderSyncLog {
|
||||
return core.HeaderSyncLog{
|
||||
ID: logID,
|
||||
HeaderID: headerID,
|
||||
Log: log,
|
||||
Transformed: false,
|
||||
}
|
||||
}
|
||||
|
||||
func buildTopics(log types.Log) pq.ByteaArray {
|
||||
var topics pq.ByteaArray
|
||||
for _, topic := range log.Topics {
|
||||
topics = append(topics, topic.Bytes())
|
||||
}
|
||||
return topics
|
||||
}
|
||||
|
||||
func createHeaderCheckedPredicateSQLForMissingHeaders(boolColumns []string) string {
|
||||
var result bytes.Buffer
|
||||
result.WriteString(" (")
|
||||
|
||||
// Loop excluding last column name
|
||||
for _, column := range boolColumns[:len(boolColumns)-1] {
|
||||
result.WriteString(fmt.Sprintf("%v=0 OR ", column))
|
||||
}
|
||||
|
||||
result.WriteString(fmt.Sprintf("%v=0)", boolColumns[len(boolColumns)-1]))
|
||||
|
||||
return result.String()
|
||||
}
|
||||
|
||||
func createHeaderCheckedPredicateSQLForRecheckedHeaders(boolColumns []string) string {
|
||||
var result bytes.Buffer
|
||||
result.WriteString(" (")
|
||||
|
||||
// Loop excluding last column name
|
||||
for _, column := range boolColumns[:len(boolColumns)-1] {
|
||||
result.WriteString(fmt.Sprintf("%v<%s OR ", column, constants.RecheckHeaderCap))
|
||||
}
|
||||
|
||||
// No trailing "OR" for the last column name
|
||||
result.WriteString(fmt.Sprintf("%v<%s)", boolColumns[len(boolColumns)-1], constants.RecheckHeaderCap))
|
||||
|
||||
return result.String()
|
||||
}
|
||||
|
||||
@@ -17,39 +17,25 @@
|
||||
package repository_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/lib/pq"
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
|
||||
"math/rand"
|
||||
"strconv"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
|
||||
shared "github.com/vulcanize/vulcanizedb/libraries/shared/repository"
|
||||
r2 "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore"
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/fakes"
|
||||
"github.com/vulcanize/vulcanizedb/test_config"
|
||||
)
|
||||
|
||||
var _ = Describe("Repository", func() {
|
||||
var (
|
||||
checkedHeadersColumn string
|
||||
db *postgres.DB
|
||||
)
|
||||
var _ = Describe("", func() {
|
||||
Describe("MarkContractWatcherHeaderCheckedInTransaction", func() {
|
||||
var (
|
||||
checkedHeadersColumn string
|
||||
db *postgres.DB
|
||||
)
|
||||
|
||||
Describe("MarkHeaderChecked", func() {
|
||||
BeforeEach(func() {
|
||||
db = test_config.NewTestDB(test_config.NewTestNode())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
checkedHeadersColumn = "test_column_checked"
|
||||
_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers
|
||||
ADD COLUMN ` + checkedHeadersColumn + ` integer`)
|
||||
@@ -61,395 +47,21 @@ var _ = Describe("Repository", func() {
|
||||
Expect(cleanupMigrateErr).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
It("marks passed column as checked for passed header", func() {
|
||||
headerRepository := repositories.NewHeaderRepository(db)
|
||||
headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
|
||||
Expect(headerErr).NotTo(HaveOccurred())
|
||||
|
||||
err := shared.MarkHeaderChecked(headerID, db, checkedHeadersColumn)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
var checkedCount int
|
||||
fetchErr := db.Get(&checkedCount, `SELECT `+checkedHeadersColumn+` FROM public.checked_headers LIMIT 1`)
|
||||
Expect(fetchErr).NotTo(HaveOccurred())
|
||||
Expect(checkedCount).To(Equal(1))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("MarkHeaderCheckedInTransaction", func() {
|
||||
BeforeEach(func() {
|
||||
db = test_config.NewTestDB(test_config.NewTestNode())
|
||||
test_config.CleanTestDB(db)
|
||||
|
||||
checkedHeadersColumn = "test_column_checked"
|
||||
_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers
|
||||
ADD COLUMN ` + checkedHeadersColumn + ` integer`)
|
||||
Expect(migrateErr).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
|
||||
Expect(cleanupMigrateErr).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
It("marks passed column as checked for passed header within a passed transaction", func() {
|
||||
It("marks passed header as checked within a passed transaction", func() {
|
||||
headerRepository := repositories.NewHeaderRepository(db)
|
||||
headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
|
||||
Expect(headerErr).NotTo(HaveOccurred())
|
||||
tx, txErr := db.Beginx()
|
||||
Expect(txErr).NotTo(HaveOccurred())
|
||||
|
||||
err := shared.MarkHeaderCheckedInTransaction(headerID, tx, checkedHeadersColumn)
|
||||
|
||||
err := repository.MarkContractWatcherHeaderCheckedInTransaction(headerID, tx, checkedHeadersColumn)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
commitErr := tx.Commit()
|
||||
Expect(commitErr).NotTo(HaveOccurred())
|
||||
var checkedCount int
|
||||
fetchErr := db.Get(&checkedCount, `SELECT `+checkedHeadersColumn+` FROM public.checked_headers LIMIT 1`)
|
||||
fetchErr := db.Get(&checkedCount, `SELECT COUNT(*) FROM public.checked_headers WHERE header_id = $1`, headerID)
|
||||
Expect(fetchErr).NotTo(HaveOccurred())
|
||||
Expect(checkedCount).To(Equal(1))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("MissingHeaders", func() {
|
||||
var (
|
||||
headerRepository datastore.HeaderRepository
|
||||
startingBlockNumber int64
|
||||
endingBlockNumber int64
|
||||
eventSpecificBlockNumber int64
|
||||
outOfRangeBlockNumber int64
|
||||
blockNumbers []int64
|
||||
headerIDs []int64
|
||||
notCheckedSQL string
|
||||
err error
|
||||
hr r2.HeaderRepository
|
||||
columnNames []string
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
db = test_config.NewTestDB(test_config.NewTestNode())
|
||||
test_config.CleanTestDB(db)
|
||||
headerRepository = repositories.NewHeaderRepository(db)
|
||||
hr = r2.NewHeaderRepository(db)
|
||||
hr.AddCheckColumns(getExpectedColumnNames())
|
||||
|
||||
columnNames, err = shared.GetCheckedColumnNames(db)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
notCheckedSQL = shared.CreateHeaderCheckedPredicateSQL(columnNames, constants.HeaderMissing)
|
||||
|
||||
startingBlockNumber = rand.Int63()
|
||||
eventSpecificBlockNumber = startingBlockNumber + 1
|
||||
endingBlockNumber = startingBlockNumber + 2
|
||||
outOfRangeBlockNumber = endingBlockNumber + 1
|
||||
|
||||
blockNumbers = []int64{startingBlockNumber, eventSpecificBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
|
||||
|
||||
headerIDs = []int64{}
|
||||
for _, n := range blockNumbers {
|
||||
headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
|
||||
headerIDs = append(headerIDs, headerID)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
test_config.CleanCheckedHeadersTable(db, getExpectedColumnNames())
|
||||
})
|
||||
|
||||
It("only treats headers as checked if the event specific logs have been checked", func() {
|
||||
//add a checked_header record, but don't mark it check for any of the columns
|
||||
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
headers, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(headers)).To(Equal(3))
|
||||
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
|
||||
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
|
||||
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
|
||||
})
|
||||
|
||||
It("only returns headers associated with the current node", func() {
|
||||
dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
|
||||
headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
|
||||
for _, n := range blockNumbers {
|
||||
_, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
nodeOneMissingHeaders, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(nodeOneMissingHeaders)).To(Equal(3))
|
||||
Expect(nodeOneMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
|
||||
Expect(nodeOneMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
|
||||
Expect(nodeOneMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
|
||||
|
||||
nodeTwoMissingHeaders, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber+10, dbTwo, notCheckedSQL)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(nodeTwoMissingHeaders)).To(Equal(3))
|
||||
Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
|
||||
Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
|
||||
})
|
||||
|
||||
It("handles an ending block of -1 ", func() {
|
||||
endingBlock := int64(-1)
|
||||
headers, err := shared.MissingHeaders(startingBlockNumber, endingBlock, db, notCheckedSQL)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(headers)).To(Equal(4))
|
||||
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber), Equal(outOfRangeBlockNumber)))
|
||||
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber), Equal(outOfRangeBlockNumber)))
|
||||
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber), Equal(outOfRangeBlockNumber)))
|
||||
Expect(headers[3].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber), Equal(outOfRangeBlockNumber)))
|
||||
|
||||
})
|
||||
|
||||
It("when a the `notCheckedSQL` argument allows for rechecks it returns headers where the checked count is less than the maximum", func() {
|
||||
columnName := columnNames[0]
|
||||
recheckedSQL := shared.CreateHeaderCheckedPredicateSQL([]string{columnName}, constants.HeaderRecheck)
|
||||
// mark every header checked at least once
|
||||
// header 4 is marked the maximum number of times, it it is not longer checked
|
||||
|
||||
maxCheckCount, intConversionErr := strconv.Atoi(constants.RecheckHeaderCap)
|
||||
Expect(intConversionErr).NotTo(HaveOccurred())
|
||||
|
||||
markHeaderOneErr := shared.MarkHeaderChecked(headerIDs[0], db, columnName)
|
||||
Expect(markHeaderOneErr).NotTo(HaveOccurred())
|
||||
markHeaderTwoErr := shared.MarkHeaderChecked(headerIDs[1], db, columnName)
|
||||
Expect(markHeaderTwoErr).NotTo(HaveOccurred())
|
||||
markHeaderThreeErr := shared.MarkHeaderChecked(headerIDs[2], db, columnName)
|
||||
Expect(markHeaderThreeErr).NotTo(HaveOccurred())
|
||||
for i := 0; i <= maxCheckCount; i++ {
|
||||
markHeaderFourErr := shared.MarkHeaderChecked(headerIDs[3], db, columnName)
|
||||
Expect(markHeaderFourErr).NotTo(HaveOccurred())
|
||||
}
|
||||
|
||||
headers, err := shared.MissingHeaders(1, -1, db, recheckedSQL)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(headers)).To(Equal(3))
|
||||
Expect(headers[0].Id).To(Or(Equal(headerIDs[0]), Equal(headerIDs[1]), Equal(headerIDs[2])))
|
||||
Expect(headers[1].Id).To(Or(Equal(headerIDs[0]), Equal(headerIDs[1]), Equal(headerIDs[2])))
|
||||
Expect(headers[2].Id).To(Or(Equal(headerIDs[0]), Equal(headerIDs[1]), Equal(headerIDs[2])))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("GetCheckedColumnNames", func() {
|
||||
It("gets the column names from checked_headers", func() {
|
||||
db := test_config.NewTestDB(test_config.NewTestNode())
|
||||
hr := r2.NewHeaderRepository(db)
|
||||
hr.AddCheckColumns(getExpectedColumnNames())
|
||||
test_config.CleanTestDB(db)
|
||||
expectedColumnNames := getExpectedColumnNames()
|
||||
actualColumnNames, err := shared.GetCheckedColumnNames(db)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(actualColumnNames).To(Equal(expectedColumnNames))
|
||||
test_config.CleanCheckedHeadersTable(db, getExpectedColumnNames())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("CreateHeaderCheckedPredicateSQL", func() {
|
||||
Describe("for headers that haven't been checked for logs", func() {
|
||||
It("generates a correct SQL string for one column", func() {
|
||||
columns := []string{"columnA"}
|
||||
expected := " (columnA=0)"
|
||||
actual := shared.CreateHeaderCheckedPredicateSQL(columns, constants.HeaderMissing)
|
||||
Expect(actual).To(Equal(expected))
|
||||
})
|
||||
|
||||
It("generates a correct SQL string for several columns", func() {
|
||||
columns := []string{"columnA", "columnB"}
|
||||
expected := " (columnA=0 OR columnB=0)"
|
||||
actual := shared.CreateHeaderCheckedPredicateSQL(columns, constants.HeaderMissing)
|
||||
Expect(actual).To(Equal(expected))
|
||||
})
|
||||
|
||||
It("defaults to FALSE when there are no columns", func() {
|
||||
expected := "FALSE"
|
||||
actual := shared.CreateHeaderCheckedPredicateSQL([]string{}, constants.HeaderMissing)
|
||||
Expect(actual).To(Equal(expected))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("for headers that are being rechecked for logs", func() {
|
||||
It("generates a correct SQL string for rechecking headers for one column", func() {
|
||||
columns := []string{"columnA"}
|
||||
expected := fmt.Sprintf(" (columnA<%s)", constants.RecheckHeaderCap)
|
||||
actual := shared.CreateHeaderCheckedPredicateSQL(columns, constants.HeaderRecheck)
|
||||
Expect(actual).To(Equal(expected))
|
||||
})
|
||||
|
||||
It("generates a correct SQL string for rechecking headers for several columns", func() {
|
||||
columns := []string{"columnA", "columnB"}
|
||||
expected := fmt.Sprintf(" (columnA<%s OR columnB<%s)", constants.RecheckHeaderCap, constants.RecheckHeaderCap)
|
||||
actual := shared.CreateHeaderCheckedPredicateSQL(columns, constants.HeaderRecheck)
|
||||
Expect(actual).To(Equal(expected))
|
||||
})
|
||||
|
||||
It("defaults to FALSE when there are no columns", func() {
|
||||
expected := "FALSE"
|
||||
actual := shared.CreateHeaderCheckedPredicateSQL([]string{}, constants.HeaderRecheck)
|
||||
Expect(actual).To(Equal(expected))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Describe("CreateHeaderSyncLogs", func() {
|
||||
var headerID int64
|
||||
|
||||
type HeaderSyncLog struct {
|
||||
ID int64
|
||||
HeaderID int64 `db:"header_id"`
|
||||
Address string
|
||||
Topics pq.ByteaArray
|
||||
Data []byte
|
||||
BlockNumber uint64 `db:"block_number"`
|
||||
BlockHash string `db:"block_hash"`
|
||||
TxHash string `db:"tx_hash"`
|
||||
TxIndex uint `db:"tx_index"`
|
||||
LogIndex uint `db:"log_index"`
|
||||
Transformed bool
|
||||
Raw []byte
|
||||
}
|
||||
|
||||
BeforeEach(func() {
|
||||
db = test_config.NewTestDB(test_config.NewTestNode())
|
||||
test_config.CleanTestDB(db)
|
||||
headerRepository := repositories.NewHeaderRepository(db)
|
||||
var headerErr error
|
||||
headerID, headerErr = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
|
||||
Expect(headerErr).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
It("writes a log to the db", func() {
|
||||
log := test_data.GenericTestLog()
|
||||
|
||||
_, err := shared.CreateLogs(headerID, []types.Log{log}, db)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
var dbLog HeaderSyncLog
|
||||
lookupErr := db.Get(&dbLog, `SELECT * FROM header_sync_logs`)
|
||||
Expect(lookupErr).NotTo(HaveOccurred())
|
||||
Expect(dbLog.ID).NotTo(BeZero())
|
||||
Expect(dbLog.HeaderID).To(Equal(headerID))
|
||||
Expect(dbLog.Address).To(Equal(log.Address.Hex()))
|
||||
Expect(dbLog.Topics[0]).To(Equal(log.Topics[0].Bytes()))
|
||||
Expect(dbLog.Topics[1]).To(Equal(log.Topics[1].Bytes()))
|
||||
Expect(dbLog.Data).To(Equal(log.Data))
|
||||
Expect(dbLog.BlockNumber).To(Equal(log.BlockNumber))
|
||||
Expect(dbLog.BlockHash).To(Equal(log.BlockHash.Hex()))
|
||||
Expect(dbLog.TxIndex).To(Equal(log.TxIndex))
|
||||
Expect(dbLog.TxHash).To(Equal(log.TxHash.Hex()))
|
||||
Expect(dbLog.LogIndex).To(Equal(log.Index))
|
||||
expectedRaw, jsonErr := log.MarshalJSON()
|
||||
Expect(jsonErr).NotTo(HaveOccurred())
|
||||
Expect(dbLog.Raw).To(MatchJSON(expectedRaw))
|
||||
Expect(dbLog.Transformed).To(BeFalse())
|
||||
})
|
||||
|
||||
It("writes several logs to the db", func() {
|
||||
log1 := test_data.GenericTestLog()
|
||||
log2 := test_data.GenericTestLog()
|
||||
logs := []types.Log{log1, log2}
|
||||
|
||||
_, err := shared.CreateLogs(headerID, logs, db)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
var count int
|
||||
lookupErr := db.Get(&count, `SELECT COUNT(*) FROM header_sync_logs`)
|
||||
Expect(lookupErr).NotTo(HaveOccurred())
|
||||
Expect(count).To(Equal(len(logs)))
|
||||
})
|
||||
|
||||
It("persists record that can be unpacked into types.Log", func() {
|
||||
// important if we want to decouple log persistence from transforming and still make use of
|
||||
// tools on types.Log like abi.Unpack
|
||||
|
||||
log := test_data.GenericTestLog()
|
||||
|
||||
_, err := shared.CreateLogs(headerID, []types.Log{log}, db)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
var dbLog HeaderSyncLog
|
||||
lookupErr := db.Get(&dbLog, `SELECT * FROM header_sync_logs`)
|
||||
Expect(lookupErr).NotTo(HaveOccurred())
|
||||
|
||||
var logTopics []common.Hash
|
||||
for _, topic := range dbLog.Topics {
|
||||
logTopics = append(logTopics, common.BytesToHash(topic))
|
||||
}
|
||||
|
||||
reconstructedLog := types.Log{
|
||||
Address: common.HexToAddress(dbLog.Address),
|
||||
Topics: logTopics,
|
||||
Data: dbLog.Data,
|
||||
BlockNumber: dbLog.BlockNumber,
|
||||
TxHash: common.HexToHash(dbLog.TxHash),
|
||||
TxIndex: dbLog.TxIndex,
|
||||
BlockHash: common.HexToHash(dbLog.BlockHash),
|
||||
Index: dbLog.LogIndex,
|
||||
Removed: false,
|
||||
}
|
||||
Expect(reconstructedLog).To(Equal(log))
|
||||
})
|
||||
|
||||
It("does not duplicate logs", func() {
|
||||
log := test_data.GenericTestLog()
|
||||
|
||||
results, err := shared.CreateLogs(headerID, []types.Log{log, log}, db)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(results)).To(Equal(1))
|
||||
var count int
|
||||
lookupErr := db.Get(&count, `SELECT COUNT(*) FROM header_sync_logs`)
|
||||
Expect(lookupErr).NotTo(HaveOccurred())
|
||||
Expect(count).To(Equal(1))
|
||||
})
|
||||
|
||||
It("returns results with log id and header id for persisted logs", func() {
|
||||
log1 := test_data.GenericTestLog()
|
||||
log2 := test_data.GenericTestLog()
|
||||
logs := []types.Log{log1, log2}
|
||||
|
||||
results, err := shared.CreateLogs(headerID, logs, db)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(results)).To(Equal(len(logs)))
|
||||
var log1ID, log2ID int64
|
||||
lookupErr := db.Get(&log1ID, `SELECT id FROM header_sync_logs WHERE data = $1`, log1.Data)
|
||||
Expect(lookupErr).NotTo(HaveOccurred())
|
||||
lookup2Err := db.Get(&log2ID, `SELECT id FROM header_sync_logs WHERE data = $1`, log2.Data)
|
||||
Expect(lookup2Err).NotTo(HaveOccurred())
|
||||
Expect(results[0].ID).To(Or(Equal(log1ID), Equal(log2ID)))
|
||||
Expect(results[1].ID).To(Or(Equal(log1ID), Equal(log2ID)))
|
||||
Expect(results[0].HeaderID).To(Equal(headerID))
|
||||
Expect(results[1].HeaderID).To(Equal(headerID))
|
||||
})
|
||||
|
||||
It("returns results with properties for persisted logs", func() {
|
||||
log1 := test_data.GenericTestLog()
|
||||
log2 := test_data.GenericTestLog()
|
||||
logs := []types.Log{log1, log2}
|
||||
|
||||
results, err := shared.CreateLogs(headerID, logs, db)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(len(results)).To(Equal(len(logs)))
|
||||
Expect(results[0].Log).To(Or(Equal(log1), Equal(log2)))
|
||||
Expect(results[1].Log).To(Or(Equal(log1), Equal(log2)))
|
||||
Expect(results[0].Transformed).To(BeFalse())
|
||||
Expect(results[1].Transformed).To(BeFalse())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
func getExpectedColumnNames() []string {
|
||||
return []string{
|
||||
"column_1",
|
||||
"column_2",
|
||||
"column_3",
|
||||
"column_4",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
)
|
||||
|
||||
type StorageRepository interface {
|
||||
Create(blockNumber int, blockHash string, metadata utils.StorageValueMetadata, value interface{}) error
|
||||
SetDB(db *postgres.DB)
|
||||
}
|
||||
Reference in New Issue
Block a user