exp backoff, short stack err

cmd/lotus-provider/proving.go

@@ -82,6 +82,7 @@ var wdPostTaskCmd = &cli.Command{
 			return xerrors.Errorf("cannot get miner id %w", err)
 		}
 		var id int64
+		retryDelay := time.Millisecond * 10
 	retryAddTask:
 		_, err = deps.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
 			err = tx.QueryRow(`INSERT INTO harmony_task (name, posted_time, added_by) VALUES ('WdPost', CURRENT_TIMESTAMP, 123) RETURNING id`).Scan(&id)
@@ -104,6 +105,8 @@ var wdPostTaskCmd = &cli.Command{
 		})
 		if err != nil {
 			if harmonydb.IsErrSerialization(err) {
+				time.Sleep(retryDelay)
+				retryDelay *= 2
 				goto retryAddTask
 			}
 			return xerrors.Errorf("writing SQL transaction: %w", err)

lib/harmony/harmonydb/userfuncs.go

@@ -124,9 +124,9 @@ type Tx struct {
 // & non-transaction calls in transactions. It only checks 20 frames.
 // Fast: This memory should all be in CPU Caches.
 func (db *DB) usedInTransaction() bool {
-	var framePtrs = (&[20]uintptr{})[:]    // 20 can be stack-local (no alloc)
+	var framePtrs = (&[20]uintptr{})[:]                   // 20 can be stack-local (no alloc)
-	runtime.Callers(3, framePtrs)          // skip past our caller.
+	framePtrs = framePtrs[:runtime.Callers(3, framePtrs)] // skip past our caller.
-	return lo.Contains(framePtrs, db.BTFP) // Unsafe read @ beginTx overlap, but 'return false' is correct there.
+	return lo.Contains(framePtrs, db.BTFP)                // Unsafe read @ beginTx overlap, but 'return false' is correct there.
 }
 
 // BeginTransaction is how you can access transactions using this library.

lib/harmony/harmonytask/task_type_handler.go

@@ -25,6 +25,7 @@ type taskTypeHandler struct {
 
 func (h *taskTypeHandler) AddTask(extra func(TaskID, *harmonydb.Tx) (bool, error)) {
 	var tID TaskID
+	retryWait := time.Millisecond * 100
 retryAddTask:
 	_, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) (bool, error) {
 		// create taskID (from DB)
@@ -46,6 +47,8 @@ retryAddTask:
 			return
 		}
 		if harmonydb.IsErrSerialization(err) {
+			time.Sleep(retryWait)
+			retryWait *= 2
 			goto retryAddTask
 		}
 		log.Error("Could not add task. AddTasFunc failed: %v", err)
@@ -165,6 +168,7 @@ top:
 
 func (h *taskTypeHandler) recordCompletion(tID TaskID, workStart time.Time, done bool, doErr error) {
 	workEnd := time.Now()
+	retryWait := time.Millisecond * 100
 retryRecordCompletion:
 	cm, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) (bool, error) {
 		var postedTime time.Time
@@ -219,6 +223,8 @@ VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, tID, h.Name, postedTime, workStart, wo
 	})
 	if err != nil {
 		if harmonydb.IsErrSerialization(err) {
+			time.Sleep(retryWait)
+			retryWait *= 2
 			goto retryRecordCompletion
 		}
 		log.Error("Could not record transaction: ", err)

storage/paths/db_index.go

@@ -180,6 +180,7 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
 		}
 	}
 
+	retryWait := time.Millisecond * 100
 retryAttachStorage:
 	// Single transaction to attach storage which is not present in the DB
 	_, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
@@ -246,6 +247,8 @@ retryAttachStorage:
 	})
 	if err != nil {
 		if harmonydb.IsErrSerialization(err) {
+			time.Sleep(retryWait)
+			retryWait *= 2
 			goto retryAttachStorage
 		}
 		return err
@@ -287,6 +290,7 @@ func (dbi *DBIndex) StorageDetach(ctx context.Context, id storiface.ID, url stri
 
 		log.Warnw("Dropping sector path endpoint", "path", id, "url", url)
 	} else {
+		retryWait := time.Millisecond * 100
 	retryDropPath:
 		// Single transaction to drop storage path and sector decls which have this as a storage path
 		_, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
@@ -305,6 +309,8 @@ func (dbi *DBIndex) StorageDetach(ctx context.Context, id storiface.ID, url stri
 		})
 		if err != nil {
 			if harmonydb.IsErrSerialization(err) {
+				time.Sleep(retryWait)
+				retryWait *= 2
 				goto retryDropPath
 			}
 			return err
@@ -380,6 +386,7 @@ func (dbi *DBIndex) StorageDeclareSector(ctx context.Context, storageID storifac
 		return xerrors.Errorf("invalid filetype")
 	}
 
+	retryWait := time.Millisecond * 100
 retryStorageDeclareSector:
 	_, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
 		var currPrimary sql.NullBool
@@ -416,6 +423,8 @@ retryStorageDeclareSector:
 	})
 	if err != nil {
 		if harmonydb.IsErrSerialization(err) {
+			time.Sleep(retryWait)
+			retryWait *= 2
 			goto retryStorageDeclareSector
 		}
 		return err