exp backoff, short stack err

This commit is contained in:
Andrew Jackson (Ajax) 2023-12-11 10:50:49 -06:00
parent 9dca4346b1
commit 96353e63ea
4 changed files with 21 additions and 3 deletions

View File

@ -82,6 +82,7 @@ var wdPostTaskCmd = &cli.Command{
return xerrors.Errorf("cannot get miner id %w", err)
}
var id int64
retryDelay := time.Millisecond * 10
retryAddTask:
_, err = deps.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
err = tx.QueryRow(`INSERT INTO harmony_task (name, posted_time, added_by) VALUES ('WdPost', CURRENT_TIMESTAMP, 123) RETURNING id`).Scan(&id)
@ -104,6 +105,8 @@ var wdPostTaskCmd = &cli.Command{
})
if err != nil {
if harmonydb.IsErrSerialization(err) {
time.Sleep(retryDelay)
retryDelay *= 2
goto retryAddTask
}
return xerrors.Errorf("writing SQL transaction: %w", err)

View File

@ -125,7 +125,7 @@ type Tx struct {
// Fast: This memory should all be in CPU Caches.
func (db *DB) usedInTransaction() bool {
var framePtrs = (&[20]uintptr{})[:] // 20 can be stack-local (no alloc)
runtime.Callers(3, framePtrs) // skip past our caller.
framePtrs = framePtrs[:runtime.Callers(3, framePtrs)] // skip past our caller.
return lo.Contains(framePtrs, db.BTFP) // Unsafe read @ beginTx overlap, but 'return false' is correct there.
}

View File

@ -25,6 +25,7 @@ type taskTypeHandler struct {
func (h *taskTypeHandler) AddTask(extra func(TaskID, *harmonydb.Tx) (bool, error)) {
var tID TaskID
retryWait := time.Millisecond * 100
retryAddTask:
_, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) (bool, error) {
// create taskID (from DB)
@ -46,6 +47,8 @@ retryAddTask:
return
}
if harmonydb.IsErrSerialization(err) {
time.Sleep(retryWait)
retryWait *= 2
goto retryAddTask
}
log.Error("Could not add task. AddTasFunc failed: %v", err)
@ -165,6 +168,7 @@ top:
func (h *taskTypeHandler) recordCompletion(tID TaskID, workStart time.Time, done bool, doErr error) {
workEnd := time.Now()
retryWait := time.Millisecond * 100
retryRecordCompletion:
cm, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) (bool, error) {
var postedTime time.Time
@ -219,6 +223,8 @@ VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, tID, h.Name, postedTime, workStart, wo
})
if err != nil {
if harmonydb.IsErrSerialization(err) {
time.Sleep(retryWait)
retryWait *= 2
goto retryRecordCompletion
}
log.Error("Could not record transaction: ", err)

View File

@ -180,6 +180,7 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
}
}
retryWait := time.Millisecond * 100
retryAttachStorage:
// Single transaction to attach storage which is not present in the DB
_, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
@ -246,6 +247,8 @@ retryAttachStorage:
})
if err != nil {
if harmonydb.IsErrSerialization(err) {
time.Sleep(retryWait)
retryWait *= 2
goto retryAttachStorage
}
return err
@ -287,6 +290,7 @@ func (dbi *DBIndex) StorageDetach(ctx context.Context, id storiface.ID, url stri
log.Warnw("Dropping sector path endpoint", "path", id, "url", url)
} else {
retryWait := time.Millisecond * 100
retryDropPath:
// Single transaction to drop storage path and sector decls which have this as a storage path
_, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
@ -305,6 +309,8 @@ func (dbi *DBIndex) StorageDetach(ctx context.Context, id storiface.ID, url stri
})
if err != nil {
if harmonydb.IsErrSerialization(err) {
time.Sleep(retryWait)
retryWait *= 2
goto retryDropPath
}
return err
@ -380,6 +386,7 @@ func (dbi *DBIndex) StorageDeclareSector(ctx context.Context, storageID storifac
return xerrors.Errorf("invalid filetype")
}
retryWait := time.Millisecond * 100
retryStorageDeclareSector:
_, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
var currPrimary sql.NullBool
@ -416,6 +423,8 @@ retryStorageDeclareSector:
})
if err != nil {
if harmonydb.IsErrSerialization(err) {
time.Sleep(retryWait)
retryWait *= 2
goto retryStorageDeclareSector
}
return err