exp backoff, short stack err

This commit is contained in:
Andrew Jackson (Ajax) 2023-12-11 10:50:49 -06:00
parent 9dca4346b1
commit 96353e63ea
4 changed files with 21 additions and 3 deletions

View File

@ -82,6 +82,7 @@ var wdPostTaskCmd = &cli.Command{
return xerrors.Errorf("cannot get miner id %w", err) return xerrors.Errorf("cannot get miner id %w", err)
} }
var id int64 var id int64
retryDelay := time.Millisecond * 10
retryAddTask: retryAddTask:
_, err = deps.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { _, err = deps.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
err = tx.QueryRow(`INSERT INTO harmony_task (name, posted_time, added_by) VALUES ('WdPost', CURRENT_TIMESTAMP, 123) RETURNING id`).Scan(&id) err = tx.QueryRow(`INSERT INTO harmony_task (name, posted_time, added_by) VALUES ('WdPost', CURRENT_TIMESTAMP, 123) RETURNING id`).Scan(&id)
@ -104,6 +105,8 @@ var wdPostTaskCmd = &cli.Command{
}) })
if err != nil { if err != nil {
if harmonydb.IsErrSerialization(err) { if harmonydb.IsErrSerialization(err) {
time.Sleep(retryDelay)
retryDelay *= 2
goto retryAddTask goto retryAddTask
} }
return xerrors.Errorf("writing SQL transaction: %w", err) return xerrors.Errorf("writing SQL transaction: %w", err)

View File

@ -124,9 +124,9 @@ type Tx struct {
// & non-transaction calls in transactions. It only checks 20 frames. // & non-transaction calls in transactions. It only checks 20 frames.
// Fast: This memory should all be in CPU Caches. // Fast: This memory should all be in CPU Caches.
func (db *DB) usedInTransaction() bool { func (db *DB) usedInTransaction() bool {
var framePtrs = (&[20]uintptr{})[:] // 20 can be stack-local (no alloc) var framePtrs = (&[20]uintptr{})[:] // 20 can be stack-local (no alloc)
runtime.Callers(3, framePtrs) // skip past our caller. framePtrs = framePtrs[:runtime.Callers(3, framePtrs)] // skip past our caller.
return lo.Contains(framePtrs, db.BTFP) // Unsafe read @ beginTx overlap, but 'return false' is correct there. return lo.Contains(framePtrs, db.BTFP) // Unsafe read @ beginTx overlap, but 'return false' is correct there.
} }
// BeginTransaction is how you can access transactions using this library. // BeginTransaction is how you can access transactions using this library.

View File

@ -25,6 +25,7 @@ type taskTypeHandler struct {
func (h *taskTypeHandler) AddTask(extra func(TaskID, *harmonydb.Tx) (bool, error)) { func (h *taskTypeHandler) AddTask(extra func(TaskID, *harmonydb.Tx) (bool, error)) {
var tID TaskID var tID TaskID
retryWait := time.Millisecond * 100
retryAddTask: retryAddTask:
_, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) (bool, error) { _, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) (bool, error) {
// create taskID (from DB) // create taskID (from DB)
@ -46,6 +47,8 @@ retryAddTask:
return return
} }
if harmonydb.IsErrSerialization(err) { if harmonydb.IsErrSerialization(err) {
time.Sleep(retryWait)
retryWait *= 2
goto retryAddTask goto retryAddTask
} }
log.Error("Could not add task. AddTasFunc failed: %v", err) log.Error("Could not add task. AddTasFunc failed: %v", err)
@ -165,6 +168,7 @@ top:
func (h *taskTypeHandler) recordCompletion(tID TaskID, workStart time.Time, done bool, doErr error) { func (h *taskTypeHandler) recordCompletion(tID TaskID, workStart time.Time, done bool, doErr error) {
workEnd := time.Now() workEnd := time.Now()
retryWait := time.Millisecond * 100
retryRecordCompletion: retryRecordCompletion:
cm, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) (bool, error) { cm, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) (bool, error) {
var postedTime time.Time var postedTime time.Time
@ -219,6 +223,8 @@ VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, tID, h.Name, postedTime, workStart, wo
}) })
if err != nil { if err != nil {
if harmonydb.IsErrSerialization(err) { if harmonydb.IsErrSerialization(err) {
time.Sleep(retryWait)
retryWait *= 2
goto retryRecordCompletion goto retryRecordCompletion
} }
log.Error("Could not record transaction: ", err) log.Error("Could not record transaction: ", err)

View File

@ -180,6 +180,7 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
} }
} }
retryWait := time.Millisecond * 100
retryAttachStorage: retryAttachStorage:
// Single transaction to attach storage which is not present in the DB // Single transaction to attach storage which is not present in the DB
_, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { _, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
@ -246,6 +247,8 @@ retryAttachStorage:
}) })
if err != nil { if err != nil {
if harmonydb.IsErrSerialization(err) { if harmonydb.IsErrSerialization(err) {
time.Sleep(retryWait)
retryWait *= 2
goto retryAttachStorage goto retryAttachStorage
} }
return err return err
@ -287,6 +290,7 @@ func (dbi *DBIndex) StorageDetach(ctx context.Context, id storiface.ID, url stri
log.Warnw("Dropping sector path endpoint", "path", id, "url", url) log.Warnw("Dropping sector path endpoint", "path", id, "url", url)
} else { } else {
retryWait := time.Millisecond * 100
retryDropPath: retryDropPath:
// Single transaction to drop storage path and sector decls which have this as a storage path // Single transaction to drop storage path and sector decls which have this as a storage path
_, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { _, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
@ -305,6 +309,8 @@ func (dbi *DBIndex) StorageDetach(ctx context.Context, id storiface.ID, url stri
}) })
if err != nil { if err != nil {
if harmonydb.IsErrSerialization(err) { if harmonydb.IsErrSerialization(err) {
time.Sleep(retryWait)
retryWait *= 2
goto retryDropPath goto retryDropPath
} }
return err return err
@ -380,6 +386,7 @@ func (dbi *DBIndex) StorageDeclareSector(ctx context.Context, storageID storifac
return xerrors.Errorf("invalid filetype") return xerrors.Errorf("invalid filetype")
} }
retryWait := time.Millisecond * 100
retryStorageDeclareSector: retryStorageDeclareSector:
_, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { _, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
var currPrimary sql.NullBool var currPrimary sql.NullBool
@ -416,6 +423,8 @@ retryStorageDeclareSector:
}) })
if err != nil { if err != nil {
if harmonydb.IsErrSerialization(err) { if harmonydb.IsErrSerialization(err) {
time.Sleep(retryWait)
retryWait *= 2
goto retryStorageDeclareSector goto retryStorageDeclareSector
} }
return err return err