Use a transaction for writing, knowngaps, and reorgs #53

Merged
abdulrabbani00 merged 4 commits from feature/52-database-write-improvements into develop 2022-06-06 13:02:43 +00:00
3 changed files with 16 additions and 10 deletions
Showing only changes of commit 385270427e - Show all commits

View File

@ -16,7 +16,7 @@ import (
var _ = Describe("Capturehistoric", func() {
Describe("Run the application in historic mode", Label("unit", "behavioral"), func() {
Context("Phase0: When we need to process a single block in the ethcl.historic_process table.", Label("now"), func() {
Context("Phase0: When we need to process a single block in the ethcl.historic_process table.", func() {
It("Successfully Process the Block", func() {
bc := setUpTest(BeaconNodeTester.TestConfig, "99")
BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)

View File

@ -319,9 +319,9 @@ func transactReorgs(tx sql.Tx, ctx context.Context, slot string, latestBlockRoot
loghelper.LogReorg(slot, latestBlockRoot).WithFields(log.Fields{
"proposedCount": proposedCount,
}).Error("Too many rows were marked as proposed!")
transactKnownGaps(tx, ctx, 1, slotNum, slotNum, err, "reorg", metrics)
transactKnownGaps(tx, ctx, 1, slotNum, slotNum, fmt.Errorf("Too many rows were marked as unproposed."), "reorg", metrics)
} else if proposedCount == 0 {
transactKnownGaps(tx, ctx, 1, slotNum, slotNum, err, "reorg", metrics)
transactKnownGaps(tx, ctx, 1, slotNum, slotNum, fmt.Errorf("Unable to find properly proposed row in DB"), "reorg", metrics)
loghelper.LogReorg(slot, latestBlockRoot).Info("Updated the row that should have been marked as proposed.")
}
@ -382,13 +382,19 @@ func updateProposed(tx sql.Tx, ctx context.Context, slot string, latestBlockRoot
// smaller chunks. For example, instead of having an entry of 1-101, if we increment the entries by 10 slots, we would
// have 10 entries as follows: 1-10, 11-20, etc...
func transactKnownGaps(tx sql.Tx, ctx context.Context, tableIncrement int, startSlot int, endSlot int, entryError error, entryProcess string, metric *BeaconClientMetrics) {
var entryErrorMsg string
if entryError == nil {
entryErrorMsg = ""
} else {
entryErrorMsg = entryError.Error()
}
if endSlot-startSlot <= tableIncrement {
kgModel := DbKnownGaps{
StartSlot: strconv.Itoa(startSlot),
EndSlot: strconv.Itoa(endSlot),
CheckedOut: false,
ReprocessingError: "",
EntryError: entryError.Error(),
EntryError: entryErrorMsg,
EntryProcess: entryProcess,
}
upsertKnownGaps(tx, ctx, kgModel, metric)
@ -413,7 +419,7 @@ func transactKnownGaps(tx sql.Tx, ctx context.Context, tableIncrement int, start
EndSlot: strconv.Itoa(tempEnd),
CheckedOut: false,
ReprocessingError: "",
EntryError: entryError.Error(),
EntryError: entryErrorMsg,
EntryProcess: entryProcess,
}
upsertKnownGaps(tx, ctx, kgModel, metric)

View File

@ -98,12 +98,12 @@ func processSlotRangeWorker(workCh <-chan int, errCh chan<- batchHistoricError,
for slot := range workCh {
log.Debug("Handling slot: ", slot)
err, errProcess := handleHistoricSlot(db, serverAddress, slot, metrics, checkDb)
errMs := batchHistoricError{
err: err,
errProcess: errProcess,
slot: slot,
}
if err != nil {
errMs := batchHistoricError{
err: err,
errProcess: errProcess,
slot: slot,
}
errCh <- errMs
}
}