pybind/mgr/rbd_support: add linear backoff to retried tasks

When a task fails but will be retried, linearly increase the
delay for the next retry up to a maximum of 5 minutes.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
(cherry picked from commit b8a4685cf9bf82a1a072e0de635a8b2a1a6973ba)

Conflicts:
        src/pybind/mgr/rbd_support/task.py
- file does not exist in nautilus: applied the changes manually

diff --git a/src/pybind/mgr/rbd_support/module.py b/src/pybind/mgr/rbd_support/module.py
index 6b1421221e124cc86ac28a67d2086ae57b9d6664..b38aa1f82b96eaf099bd3106b937b4dda0543d53 100644 (file)
--- a/src/pybind/mgr/rbd_support/module.py
+++ b/src/pybind/mgr/rbd_support/module.py
@@ -53,6 +53,7 @@ TASK_SEQUENCE = "sequence"
 TASK_ID = "id"
 TASK_REFS = "refs"
 TASK_MESSAGE = "message"
+TASK_RETRY_ATTEMPTS = "retry_attempts"
 TASK_RETRY_TIME = "retry_time"
 TASK_IN_PROGRESS = "in_progress"
 TASK_PROGRESS = "progress"
@@ -79,6 +80,7 @@ VALID_TASK_ACTIONS = [TASK_REF_ACTION_FLATTEN,
                       TASK_REF_ACTION_MIGRATION_ABORT]
 
 TASK_RETRY_INTERVAL = timedelta(seconds=30)
+TASK_MAX_RETRY_INTERVAL = timedelta(seconds=300)
 MAX_COMPLETED_TASKS = 50
 
 
@@ -553,6 +555,7 @@ class Task:
         self.task_id = task_id
         self.message = message
         self.refs = refs
+        self.retry_attempts = 0
         self.retry_time = None
         self.in_progress = False
         self.progress = 0.0
@@ -580,6 +583,8 @@ class Task:
              TASK_MESSAGE: self.message,
              TASK_REFS: self.refs
              }
+        if self.retry_attempts:
+            d[TASK_RETRY_ATTEMPTS] = self.retry_attempts
         if self.retry_time:
             d[TASK_RETRY_TIME] = self.retry_time.isoformat()
         if self.in_progress:
@@ -880,7 +885,10 @@ class TaskHandler:
 
         finally:
             task.in_progress = False
-            task.retry_time = datetime.now() + TASK_RETRY_INTERVAL
+            task.retry_attempts += 1
+            task.retry_time = datetime.now() + min(
+                TASK_RETRY_INTERVAL * task.retry_attempts,
+                TASK_MAX_RETRY_INTERVAL)
 
     def progress_callback(self, task, current, total):
         progress = float(current) / float(total)