From: Jason Dillaman <dillaman@redhat.com>
Date: Thu, 19 Nov 2020 20:25:52 +0000 (-0500)
Subject: pybind/mgr/rbd_support: add linear backoff to retried tasks
X-Git-Tag: v16.1.0~519^2~2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=b8a4685cf9bf82a1a072e0de635a8b2a1a6973ba;p=ceph.git

pybind/mgr/rbd_support: add linear backoff to retried tasks

When a task fails but will be retried, linearly increase the
delay for the next retry up to a maximum of 5 minutes.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---

diff --git a/src/pybind/mgr/rbd_support/task.py b/src/pybind/mgr/rbd_support/task.py
index c17ffa102f7d..12a4a3c06dee 100644
--- a/src/pybind/mgr/rbd_support/task.py
+++ b/src/pybind/mgr/rbd_support/task.py
@@ -21,6 +21,7 @@ TASK_SEQUENCE = "sequence"
 TASK_ID = "id"
 TASK_REFS = "refs"
 TASK_MESSAGE = "message"
+TASK_RETRY_ATTEMPTS = "retry_attempts"
 TASK_RETRY_TIME = "retry_time"
 TASK_IN_PROGRESS = "in_progress"
 TASK_PROGRESS = "progress"
@@ -47,6 +48,7 @@ VALID_TASK_ACTIONS = [TASK_REF_ACTION_FLATTEN,
                       TASK_REF_ACTION_MIGRATION_ABORT]
 
 TASK_RETRY_INTERVAL = timedelta(seconds=30)
+TASK_MAX_RETRY_INTERVAL = timedelta(seconds=300)
 MAX_COMPLETED_TASKS = 50
 
 
@@ -71,6 +73,7 @@ class Task:
         self.task_id = task_id
         self.message = message
         self.refs = refs
+        self.retry_attempts = 0
         self.retry_time = None
         self.in_progress = False
         self.progress = 0.0
@@ -98,6 +101,8 @@ class Task:
              TASK_MESSAGE: self.message,
              TASK_REFS: self.refs
              }
+        if self.retry_attempts:
+            d[TASK_RETRY_ATTEMPTS] = self.retry_attempts
         if self.retry_time:
             d[TASK_RETRY_TIME] = self.retry_time.isoformat()
         if self.in_progress:
@@ -398,7 +403,10 @@ class TaskHandler:
 
         finally:
             task.in_progress = False
-            task.retry_time = datetime.now() + TASK_RETRY_INTERVAL
+            task.retry_attempts += 1
+            task.retry_time = datetime.now() + min(
+                TASK_RETRY_INTERVAL * task.retry_attempts,
+                TASK_MAX_RETRY_INTERVAL)
 
     def progress_callback(self, task, current, total):
         progress = float(current) / float(total)