cerc-io/ipld-eth-beacon-indexer
19
0
Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases 9 Wiki Activity

Merge pull request #64 from vulcanize/feature/bug-remove-processed-slots

Browse Source 
Send slots to processedCh and remove
This commit is contained in:
Abdul Rabbani 2022-06-14 09:35:28 -04:00 committed by GitHub
commit d5a9f999b3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pkg/beaconclient/capturehistoric.go
View File

@ -100,6 +100,8 @@ func handleBatchProcess(ctx context.Context, maxWorkers int, bp BatchProcessing,
// Start workers // Start workers
for w := 1; w <= maxWorkers; w++ { for w := 1; w <= maxWorkers; w++ {
log.WithFields(log.Fields{"maxWorkers": maxWorkers}).Debug("Starting batch processing workers") log.WithFields(log.Fields{"maxWorkers": maxWorkers}).Debug("Starting batch processing workers")
// Pass in function to increment metric! KnownGapProcessing or HistoricProcessing
go processSlotRangeWorker(ctx, workCh, errCh, db, serverEndpoint, metrics, checkDb) go processSlotRangeWorker(ctx, workCh, errCh, db, serverEndpoint, metrics, checkDb)
} }
@ -125,6 +127,7 @@ func handleBatchProcess(ctx context.Context, maxWorkers int, bp BatchProcessing,
} else if slots.startSlot == slots.endSlot { } else if slots.startSlot == slots.endSlot {
log.WithField("slot", slots.startSlot).Debug("Added new slot to workCh") log.WithField("slot", slots.startSlot).Debug("Added new slot to workCh")
workCh <- slots.startSlot workCh <- slots.startSlot
processedCh <- slots
} else { } else {
for i := slots.startSlot; i <= slots.endSlot; i++ { for i := slots.startSlot; i <= slots.endSlot; i++ {
workCh <- i workCh <- i

10
pkg/beaconclient/processhistoric.go
View File

@ -237,8 +237,11 @@ func removeRowPostProcess(ctx context.Context, db sql.Database, processCh <-chan
case slots := <-processCh: case slots := <-processCh:
// Make sure the start and end slot exist in the slots table. // Make sure the start and end slot exist in the slots table.
go func() { go func() {
finishedProcess := false log.WithFields(log.Fields{
for !finishedProcess { "startSlot": slots.startSlot,
"endSlot": slots.endSlot,
}).Debug("Starting to check to see if the following slots have been processed")
for {
isStartProcess, err := isSlotProcessed(db, checkProcessedStmt, strconv.Itoa(slots.startSlot)) isStartProcess, err := isSlotProcessed(db, checkProcessedStmt, strconv.Itoa(slots.startSlot))
if err != nil { if err != nil {
errCh <- err errCh <- err
@ -248,8 +251,9 @@ func removeRowPostProcess(ctx context.Context, db sql.Database, processCh <-chan
errCh <- err errCh <- err
} }
if isStartProcess && isEndProcess { if isStartProcess && isEndProcess {
finishedProcess = true break
} }
time.Sleep(1000 * time.Millisecond)
} }
_, err := db.Exec(context.Background(), removeStmt, strconv.Itoa(slots.startSlot), strconv.Itoa(slots.endSlot)) _, err := db.Exec(context.Background(), removeStmt, strconv.Itoa(slots.startSlot), strconv.Itoa(slots.endSlot))