deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

fetch event logs in batches- all events for a contract address at a given block height are fetched together, different contract events still fetched separately

Browse Source
This commit is contained in:
Ian Norden
2018-12-21 10:33:31 -06:00
parent e390a97502
commit d188329661
11 changed files with 354 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/omni/light/repository/header_repository.go
+79 -3
View File
@@ -29,11 +29,14 @@ import (
const columnCacheSize = 1000
type HeaderRepository interface {
AddCheckColumn(eventID string) error
AddCheckColumn(id string) error
AddCheckColumns(ids []string) error
MarkHeaderChecked(headerID int64, eventID string) error
MarkHeadersChecked(headers []core.Header, ids []string) error
MarkHeaderCheckedForAll(headerID int64, ids []string) error
MarkHeadersCheckedForAll(headers []core.Header, ids []string) error
MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, eventID string) ([]core.Header, error)
MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error)
MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error)
CheckCache(key string) (interface{}, bool)
}
@@ -70,6 +73,29 @@ func (r *headerRepository) AddCheckColumn(id string) error {
return nil
}
func (r *headerRepository) AddCheckColumns(ids []string) error {
var err error
baseQuery := "ALTER TABLE public.checked_headers"
input := make([]string, 0, len(ids))
for _, id := range ids {
_, ok := r.columns.Get(id)
if !ok {
baseQuery += " ADD COLUMN IF NOT EXISTS " + id + " BOOLEAN NOT NULL DEFAULT FALSE,"
input = append(input, id)
}
}
if len(input) > 0 {
_, err = r.db.Exec(baseQuery[:len(baseQuery)-1])
if err == nil {
for _, id := range input {
r.columns.Add(id, true)
}
}
}
return err
}
func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error {
_, err := r.db.Exec(`INSERT INTO public.checked_headers (header_id, `+id+`)
VALUES ($1, $2)
@@ -79,7 +105,26 @@ func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error {
return err
}
func (r *headerRepository) MarkHeadersChecked(headers []core.Header, ids []string) error {
func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) error {
pgStr := "INSERT INTO public.checked_headers (header_id, "
for _, id := range ids {
pgStr += id + ", "
}
pgStr = pgStr[:len(pgStr)-2] + ") VALUES ($1, "
for i := 0; i < len(ids); i++ {
pgStr += "true, "
}
pgStr = pgStr[:len(pgStr)-2] + ") ON CONFLICT (header_id) DO UPDATE SET "
for _, id := range ids {
pgStr += fmt.Sprintf("%s = true, ", id)
}
pgStr = pgStr[:len(pgStr)-2]
_, err := r.db.Exec(pgStr, headerID)
return err
}
func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error {
tx, err := r.db.Begin()
if err != nil {
return err
@@ -136,10 +181,41 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber
return result, err
}
func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error) {
var result []core.Header
var query string
var err error
baseQuery := `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL`
for _, id := range ids {
baseQuery += ` OR ` + id + ` IS FALSE`
}
if endingBlockNumber == -1 {
endStr := `) AND headers.block_number >= $1
AND headers.eth_node_fingerprint = $2
ORDER BY headers.block_number`
query = baseQuery + endStr
err = r.db.Select(&result, query, startingBlockNumber, r.db.Node.ID)
} else {
endStr := `) AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3
ORDER BY headers.block_number`
query = baseQuery + endStr
err = r.db.Select(&result, query, startingBlockNumber, endingBlockNumber, r.db.Node.ID)
}
return result, err
}
func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error) {
var result []core.Header
var query string
var err error
baseQuery := `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id IS NOT NULL`
pkg/omni/light/repository/header_repository_test.go
+121 -32
View File
@@ -34,13 +34,21 @@ var _ = Describe("Repository", func() {
var db *postgres.DB
var omniHeaderRepo repository.HeaderRepository // omni/light header repository
var coreHeaderRepo repositories.HeaderRepository // pkg/datastore header repository
var eventID, query string
var eventIDs = []string{
"eventName_contractAddr",
"eventName_contractAddr2",
"eventName_contractAddr3",
}
var methodIDs = []string{
"methodName_contractAddr",
"methodName_contractAddr2",
"methodName_contractAddr3",
}
BeforeEach(func() {
db, _ = test_helpers.SetupDBandBC()
omniHeaderRepo = repository.NewHeaderRepository(db)
coreHeaderRepo = repositories.NewHeaderRepository(db)
eventID = "eventName_contractAddr"
})
AfterEach(func() {
@@ -49,11 +57,11 @@ var _ = Describe("Repository", func() {
Describe("AddCheckColumn", func() {
It("Creates a column for the given eventID to mark if the header has been checked for that event", func() {
query = fmt.Sprintf("SELECT %s FROM checked_headers", eventID)
query := fmt.Sprintf("SELECT %s FROM checked_headers", eventIDs[0])
_, err := db.Exec(query)
Expect(err).To(HaveOccurred())
err = omniHeaderRepo.AddCheckColumn(eventID)
err = omniHeaderRepo.AddCheckColumn(eventIDs[0])
Expect(err).ToNot(HaveOccurred())
_, err = db.Exec(query)
@@ -61,35 +69,68 @@ var _ = Describe("Repository", func() {
})
It("Caches column it creates so that it does not need to repeatedly query the database to check for it's existence", func() {
_, ok := omniHeaderRepo.CheckCache(eventID)
_, ok := omniHeaderRepo.CheckCache(eventIDs[0])
Expect(ok).To(Equal(false))
err := omniHeaderRepo.AddCheckColumn(eventID)
err := omniHeaderRepo.AddCheckColumn(eventIDs[0])
Expect(err).ToNot(HaveOccurred())
v, ok := omniHeaderRepo.CheckCache(eventID)
v, ok := omniHeaderRepo.CheckCache(eventIDs[0])
Expect(ok).To(Equal(true))
Expect(v).To(Equal(true))
})
})
Describe("AddCheckColumns", func() {
It("Creates a column for the given eventIDs to mark if the header has been checked for those events", func() {
for _, id := range eventIDs {
_, err := db.Exec(fmt.Sprintf("SELECT %s FROM checked_headers", id))
Expect(err).To(HaveOccurred())
}
err := omniHeaderRepo.AddCheckColumns(eventIDs)
Expect(err).ToNot(HaveOccurred())
for _, id := range eventIDs {
_, err := db.Exec(fmt.Sprintf("SELECT %s FROM checked_headers", id))
Expect(err).ToNot(HaveOccurred())
}
})
It("Caches columns it creates so that it does not need to repeatedly query the database to check for it's existence", func() {
for _, id := range eventIDs {
_, ok := omniHeaderRepo.CheckCache(id)
Expect(ok).To(Equal(false))
}
err := omniHeaderRepo.AddCheckColumns(eventIDs)
Expect(err).ToNot(HaveOccurred())
for _, id := range eventIDs {
v, ok := omniHeaderRepo.CheckCache(id)
Expect(ok).To(Equal(true))
Expect(v).To(Equal(true))
}
})
})
Describe("MissingHeaders", func() {
It("Returns all unchecked headers for the given eventID", func() {
addHeaders(coreHeaderRepo)
err := omniHeaderRepo.AddCheckColumn(eventID)
err := omniHeaderRepo.AddCheckColumn(eventIDs[0])
Expect(err).ToNot(HaveOccurred())
missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventID)
missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0])
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(3))
})
It("Returns unchecked headers in ascending order", func() {
addHeaders(coreHeaderRepo)
err := omniHeaderRepo.AddCheckColumn(eventID)
err := omniHeaderRepo.AddCheckColumn(eventIDs[0])
Expect(err).ToNot(HaveOccurred())
missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventID)
missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0])
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(3))
@@ -103,7 +144,7 @@ var _ = Describe("Repository", func() {
It("Fails if eventID does not yet exist in check_headers table", func() {
addHeaders(coreHeaderRepo)
err := omniHeaderRepo.AddCheckColumn(eventID)
err := omniHeaderRepo.AddCheckColumn(eventIDs[0])
Expect(err).ToNot(HaveOccurred())
_, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, "notEventId")
@@ -111,31 +152,69 @@ var _ = Describe("Repository", func() {
})
})
Describe("MissingHeadersForAll", func() { // HERE
It("Returns all headers that have not been checked for all of the ids provided", func() {
addHeaders(coreHeaderRepo)
err := omniHeaderRepo.AddCheckColumns(eventIDs)
Expect(err).ToNot(HaveOccurred())
missingHeaders, err := omniHeaderRepo.MissingHeadersForAll(6194630, 6194635, eventIDs)
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(3))
err = omniHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[0])
Expect(err).ToNot(HaveOccurred())
missingHeaders, err = omniHeaderRepo.MissingHeadersForAll(6194630, 6194635, eventIDs)
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(3))
err = omniHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[1])
Expect(err).ToNot(HaveOccurred())
err = omniHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[2])
Expect(err).ToNot(HaveOccurred())
missingHeaders, err = omniHeaderRepo.MissingHeadersForAll(6194630, 6194635, eventIDs)
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(2))
})
It("Fails if one of the eventIDs does not yet exist in check_headers table", func() {
addHeaders(coreHeaderRepo)
err := omniHeaderRepo.AddCheckColumns(eventIDs)
Expect(err).ToNot(HaveOccurred())
badEventIDs := append(eventIDs, "notEventId")
_, err = omniHeaderRepo.MissingHeadersForAll(6194630, 6194635, badEventIDs)
Expect(err).To(HaveOccurred())
})
})
Describe("MarkHeaderChecked", func() {
It("Marks the header checked for the given eventID", func() {
addHeaders(coreHeaderRepo)
err := omniHeaderRepo.AddCheckColumn(eventID)
err := omniHeaderRepo.AddCheckColumn(eventIDs[0])
Expect(err).ToNot(HaveOccurred())
missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventID)
missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0])
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(3))
headerID := missingHeaders[0].Id
err = omniHeaderRepo.MarkHeaderChecked(headerID, eventID)
err = omniHeaderRepo.MarkHeaderChecked(headerID, eventIDs[0])
Expect(err).ToNot(HaveOccurred())
missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, eventID)
missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0])
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(2))
})
It("Fails if eventID does not yet exist in check_headers table", func() {
addHeaders(coreHeaderRepo)
err := omniHeaderRepo.AddCheckColumn(eventID)
err := omniHeaderRepo.AddCheckColumn(eventIDs[0])
Expect(err).ToNot(HaveOccurred())
missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventID)
missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0])
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(3))
@@ -143,13 +222,33 @@ var _ = Describe("Repository", func() {
err = omniHeaderRepo.MarkHeaderChecked(headerID, "notEventId")
Expect(err).To(HaveOccurred())
missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, eventID)
missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0])
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(3))
})
})
Describe("MarkHeadersChecked", func() {
Describe("MarkHeaderCheckedForAll", func() {
It("Marks the header checked for all provided column ids", func() {
addHeaders(coreHeaderRepo)
err := omniHeaderRepo.AddCheckColumns(eventIDs)
Expect(err).ToNot(HaveOccurred())
missingHeaders, err := omniHeaderRepo.MissingHeadersForAll(6194630, 6194635, eventIDs)
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(3))
headerID := missingHeaders[0].Id
err = omniHeaderRepo.MarkHeaderCheckedForAll(headerID, eventIDs)
Expect(err).ToNot(HaveOccurred())
missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0])
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(2))
})
})
Describe("MarkHeadersCheckedForAll", func() {
It("Marks the headers checked for all provided column ids", func() {
addHeaders(coreHeaderRepo)
methodIDs := []string{
@@ -167,7 +266,7 @@ var _ = Describe("Repository", func() {
Expect(len(missingHeaders)).To(Equal(3))
}
err := omniHeaderRepo.MarkHeadersChecked(missingHeaders, methodIDs)
err := omniHeaderRepo.MarkHeadersCheckedForAll(missingHeaders, methodIDs)
Expect(err).ToNot(HaveOccurred())
for _, id := range methodIDs {
missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, id)
@@ -180,16 +279,6 @@ var _ = Describe("Repository", func() {
Describe("MissingMethodsCheckedEventsIntersection", func() {
It("Returns headers that have been checked for all the provided events but have not been checked for all the provided methods", func() {
addHeaders(coreHeaderRepo)
eventIDs := []string{
eventID,
"eventName_contractAddr2",
"eventName_contractAddr3",
}
methodIDs := []string{
"methodName_contractAddr",
"methodName_contractAddr2",
"methodName_contractAddr3",
}
for i, id := range eventIDs {
err := omniHeaderRepo.AddCheckColumn(id)
Expect(err).ToNot(HaveOccurred())
@@ -197,7 +286,7 @@ var _ = Describe("Repository", func() {
Expect(err).ToNot(HaveOccurred())
}
missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventID)
missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0])
Expect(err).ToNot(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(3))