deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

finish method polling with hash or address type argument values

Browse Source 
collected from watched events; 'MissingMethodsCheckedEventsIntersection'
method to find headers which have been checked for each event of
interest but methods have not yet been polled at that header; tests for
new features; travis ci go version 1.9 -> 1.11 ; consolidate omniWatcher and lightOmniWatcher into single command with light as default
This commit is contained in:
Ian Norden
2018-12-21 10:33:31 -06:00
parent 0a59f06cac
commit e390a97502
25 changed files with 1022 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/omni/light/repository/header_repository.go
+77 -10
View File
@@ -18,6 +18,7 @@ package repository
import (
"database/sql"
"fmt"
"github.com/hashicorp/golang-lru"
@@ -30,7 +31,9 @@ const columnCacheSize = 1000
type HeaderRepository interface {
AddCheckColumn(eventID string) error
MarkHeaderChecked(headerID int64, eventID string) error
MarkHeadersChecked(headers []core.Header, ids []string) error
MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, eventID string) ([]core.Header, error)
MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error)
CheckCache(key string) (interface{}, bool)
}
@@ -47,36 +50,66 @@ func NewHeaderRepository(db *postgres.DB) *headerRepository {
}
}
func (r *headerRepository) AddCheckColumn(eventID string) error {
func (r *headerRepository) AddCheckColumn(id string) error {
// Check cache to see if column already exists before querying pg
_, ok := r.columns.Get(eventID)
_, ok := r.columns.Get(id)
if ok {
return nil
}
pgStr := "ALTER TABLE public.checked_headers ADD COLUMN IF NOT EXISTS "
pgStr = pgStr + eventID + " BOOLEAN NOT NULL DEFAULT FALSE"
pgStr = pgStr + id + " BOOLEAN NOT NULL DEFAULT FALSE"
_, err := r.db.Exec(pgStr)
if err != nil {
return err
}
// Add column name to cache
r.columns.Add(eventID, true)
r.columns.Add(id, true)
return nil
}
func (r *headerRepository) MarkHeaderChecked(headerID int64, eventID string) error {
_, err := r.db.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`)
func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error {
_, err := r.db.Exec(`INSERT INTO public.checked_headers (header_id, `+id+`)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET `+eventID+` = $2`, headerID, true)
UPDATE SET `+id+` = $2`, headerID, true)
return err
}
func (r *headerRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, eventID string) ([]core.Header, error) {
func (r *headerRepository) MarkHeadersChecked(headers []core.Header, ids []string) error {
tx, err := r.db.Begin()
if err != nil {
return err
}
for _, header := range headers {
pgStr := "INSERT INTO public.checked_headers (header_id, "
for _, id := range ids {
pgStr += id + ", "
}
pgStr = pgStr[:len(pgStr)-2] + ") VALUES ($1, "
for i := 0; i < len(ids); i++ {
pgStr += "true, "
}
pgStr = pgStr[:len(pgStr)-2] + ") ON CONFLICT (header_id) DO UPDATE SET "
for _, id := range ids {
pgStr += fmt.Sprintf("%s = true, ", id)
}
pgStr = pgStr[:len(pgStr)-2]
_, err = tx.Exec(pgStr, header.Id)
if err != nil {
tx.Rollback()
return err
}
}
return tx.Commit()
}
func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64, id string) ([]core.Header, error) {
var result []core.Header
var query string
var err error
@@ -84,7 +117,7 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber int64, endingBlock
if endingBlockNumber == -1 {
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR ` + eventID + ` IS FALSE)
WHERE (header_id ISNULL OR ` + id + ` IS FALSE)
AND headers.block_number >= $1
AND headers.eth_node_fingerprint = $2
ORDER BY headers.block_number`
@@ -92,7 +125,7 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber int64, endingBlock
} else {
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR ` + eventID + ` IS FALSE)
WHERE (header_id ISNULL OR ` + id + ` IS FALSE)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3
@@ -103,6 +136,40 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber int64, endingBlock
return result, err
}
func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error) {
var result []core.Header
var query string
var err error
baseQuery := `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id IS NOT NULL`
for _, id := range eventIds {
baseQuery += ` AND ` + id + ` IS TRUE`
}
baseQuery += `) AND (`
for _, id := range methodIds {
baseQuery += id + ` IS FALSE AND `
}
baseQuery = baseQuery[:len(baseQuery)-5] + `) `
if endingBlockNumber == -1 {
endStr := `AND headers.block_number >= $1
AND headers.eth_node_fingerprint = $2
ORDER BY headers.block_number`
query = baseQuery + endStr
err = r.db.Select(&result, query, startingBlockNumber, r.db.Node.ID)
} else {
endStr := `AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3
ORDER BY headers.block_number`
query = baseQuery + endStr
err = r.db.Select(&result, query, startingBlockNumber, endingBlockNumber, r.db.Node.ID)
}
return result, err
}
func (r *headerRepository) CheckCache(key string) (interface{}, bool) {
return r.columns.Get(key)
}
pkg/omni/light/repository/header_repository_test.go
+120 -33
View File
@@ -18,6 +18,7 @@ package repository_test
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@@ -31,14 +32,14 @@ import (
var _ = Describe("Repository", func() {
var db *postgres.DB
var r repository.HeaderRepository
var headerRepository repositories.HeaderRepository
var omniHeaderRepo repository.HeaderRepository // omni/light header repository
var coreHeaderRepo repositories.HeaderRepository // pkg/datastore header repository
var eventID, query string
BeforeEach(func() {
db, _ = test_helpers.SetupDBandBC()
r = repository.NewHeaderRepository(db)
headerRepository = repositories.NewHeaderRepository(db)
omniHeaderRepo = repository.NewHeaderRepository(db)
coreHeaderRepo = repositories.NewHeaderRepository(db)
eventID = "eventName_contractAddr"
})
@@ -52,7 +53,7 @@ var _ = Describe("Repository", func() {
_, err := db.Exec(query)
Expect(err).To(HaveOccurred())
err = r.AddCheckColumn(eventID)
err = omniHeaderRepo.AddCheckColumn(eventID)
Expect(err).ToNot(HaveOccurred())
_, err = db.Exec(query)
@@ -60,13 +61,13 @@ var _ = Describe("Repository", func() {
})
It("Caches column it creates so that it does not need to repeatedly query the database to check for it's existence", func() {
_, ok := r.CheckCache(eventID)
_, ok := omniHeaderRepo.CheckCache(eventID)
Expect(ok).To(Equal(false))
err := r.AddCheckColumn(eventID)
err := omniHeaderRepo.AddCheckColumn(eventID)
Expect(err).ToNot(HaveOccurred())
v, ok := r.CheckCache(eventID)
v, ok := omniHeaderRepo.CheckCache(eventID)
Expect(ok).To(Equal(true))
Expect(v).To(Equal(true))
})
@@ -74,68 +75,154 @@ var _ = Describe("Repository", func() {
Describe("MissingHeaders", func() {
It("Returns all unchecked headers for the given eventID", func() {
headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
headerRepository.CreateOrUpdateHeader(mocks.MockHeader2)
headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
err := r.AddCheckColumn(eventID)
addHeaders(coreHeaderRepo)
err := omniHeaderRepo.AddCheckColumn(eventID)
Expect(err).ToNot(HaveOccurred())
missingHeaders, err := r.MissingHeaders(6194630, 6194635, eventID)
missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventID)
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(3))
})
It("Fails if eventID does not yet exist in check_headers table", func() {
headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
headerRepository.CreateOrUpdateHeader(mocks.MockHeader2)
headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
err := r.AddCheckColumn(eventID)
It("Returns unchecked headers in ascending order", func() {
addHeaders(coreHeaderRepo)
err := omniHeaderRepo.AddCheckColumn(eventID)
Expect(err).ToNot(HaveOccurred())
_, err = r.MissingHeaders(6194630, 6194635, "notEventId")
missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventID)
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(3))
h1 := missingHeaders[0]
h2 := missingHeaders[1]
h3 := missingHeaders[2]
Expect(h1.BlockNumber).To(Equal(int64(6194632)))
Expect(h2.BlockNumber).To(Equal(int64(6194633)))
Expect(h3.BlockNumber).To(Equal(int64(6194634)))
})
It("Fails if eventID does not yet exist in check_headers table", func() {
addHeaders(coreHeaderRepo)
err := omniHeaderRepo.AddCheckColumn(eventID)
Expect(err).ToNot(HaveOccurred())
_, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, "notEventId")
Expect(err).To(HaveOccurred())
})
})
Describe("MarkHeaderChecked", func() {
It("Marks the header checked for the given eventID", func() {
headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
headerRepository.CreateOrUpdateHeader(mocks.MockHeader2)
headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
err := r.AddCheckColumn(eventID)
addHeaders(coreHeaderRepo)
err := omniHeaderRepo.AddCheckColumn(eventID)
Expect(err).ToNot(HaveOccurred())
missingHeaders, err := r.MissingHeaders(6194630, 6194635, eventID)
missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventID)
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(3))
headerID := missingHeaders[0].Id
err = r.MarkHeaderChecked(headerID, eventID)
err = omniHeaderRepo.MarkHeaderChecked(headerID, eventID)
Expect(err).ToNot(HaveOccurred())
missingHeaders, err = r.MissingHeaders(6194630, 6194635, eventID)
missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, eventID)
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(2))
})
It("Fails if eventID does not yet exist in check_headers table", func() {
headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
headerRepository.CreateOrUpdateHeader(mocks.MockHeader2)
headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
err := r.AddCheckColumn(eventID)
addHeaders(coreHeaderRepo)
err := omniHeaderRepo.AddCheckColumn(eventID)
Expect(err).ToNot(HaveOccurred())
missingHeaders, err := r.MissingHeaders(6194630, 6194635, eventID)
missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventID)
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(3))
headerID := missingHeaders[0].Id
err = r.MarkHeaderChecked(headerID, "notEventId")
err = omniHeaderRepo.MarkHeaderChecked(headerID, "notEventId")
Expect(err).To(HaveOccurred())
missingHeaders, err = r.MissingHeaders(6194630, 6194635, eventID)
missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, eventID)
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(3))
})
})
Describe("MarkHeadersChecked", func() {
It("Marks the headers checked for all provided column ids", func() {
addHeaders(coreHeaderRepo)
methodIDs := []string{
"methodName_contractAddr",
"methodName_contractAddr2",
"methodName_contractAddr3",
}
var missingHeaders []core.Header
for _, id := range methodIDs {
err := omniHeaderRepo.AddCheckColumn(id)
Expect(err).ToNot(HaveOccurred())
missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, id)
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(3))
}
err := omniHeaderRepo.MarkHeadersChecked(missingHeaders, methodIDs)
Expect(err).ToNot(HaveOccurred())
for _, id := range methodIDs {
missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, id)
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(0))
}
})
})
Describe("MissingMethodsCheckedEventsIntersection", func() {
It("Returns headers that have been checked for all the provided events but have not been checked for all the provided methods", func() {
addHeaders(coreHeaderRepo)
eventIDs := []string{
eventID,
"eventName_contractAddr2",
"eventName_contractAddr3",
}
methodIDs := []string{
"methodName_contractAddr",
"methodName_contractAddr2",
"methodName_contractAddr3",
}
for i, id := range eventIDs {
err := omniHeaderRepo.AddCheckColumn(id)
Expect(err).ToNot(HaveOccurred())
err = omniHeaderRepo.AddCheckColumn(methodIDs[i])
Expect(err).ToNot(HaveOccurred())
}
missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventID)
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(3))
headerID := missingHeaders[0].Id
headerID2 := missingHeaders[1].Id
for i, id := range eventIDs {
err = omniHeaderRepo.MarkHeaderChecked(headerID, id)
Expect(err).ToNot(HaveOccurred())
err = omniHeaderRepo.MarkHeaderChecked(headerID2, id)
Expect(err).ToNot(HaveOccurred())
err = omniHeaderRepo.MarkHeaderChecked(headerID, methodIDs[i])
Expect(err).ToNot(HaveOccurred())
}
intersectionHeaders, err := omniHeaderRepo.MissingMethodsCheckedEventsIntersection(6194630, 6194635, methodIDs, eventIDs)
Expect(err).ToNot(HaveOccurred())
Expect(len(intersectionHeaders)).To(Equal(1))
Expect(intersectionHeaders[0].Id).To(Equal(headerID2))
})
})
})
func addHeaders(coreHeaderRepo repositories.HeaderRepository) {
coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader1)
coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader2)
coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader3)
}