From 5ec88d0e91579ec9f304b985d3808d43339ea644 Mon Sep 17 00:00:00 2001
From: Jason Dillaman <dillaman@redhat.com>
Date: Thu, 15 Nov 2018 12:19:37 +0530
Subject: [PATCH] librbd: retry mirror disable if parent/clone ordering causes
 issues

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---
 src/librbd/api/Mirror.cc | 95 ++++++++++++++++++++++++----------------
 1 file changed, 57 insertions(+), 38 deletions(-)

diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc
index 66afdb565d3..9e3619b1223 100644
--- a/src/librbd/api/Mirror.cc
+++ b/src/librbd/api/Mirror.cc
@@ -647,48 +647,67 @@ int Mirror<I>::mode_set(librados::IoCtx& io_ctx,
       }
     }
   } else if (next_mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) {
-    std::set<std::string> image_ids;
-    r = list_mirror_images(io_ctx, image_ids);
-    if (r < 0) {
-      lderr(cct) << "failed listing images: " << cpp_strerror(r) << dendl;
-      return r;
-    }
+    while (true) {
+      bool retry_busy = false;
+      bool pending_busy = false;
 
-    for (const auto& img_id : image_ids) {
-      if (current_mirror_mode == cls::rbd::MIRROR_MODE_IMAGE) {
-        cls::rbd::MirrorImage mirror_image;
-        r = cls_client::mirror_image_get(&io_ctx, img_id, &mirror_image);
-        if (r < 0 && r != -ENOENT) {
-          lderr(cct) << "failed to retrieve mirroring state for image id "
-                     << img_id << ": " << cpp_strerror(r) << dendl;
-          return r;
-        }
-        if (mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_ENABLED) {
-          lderr(cct) << "failed to disable mirror mode: there are still "
-                     << "images with mirroring enabled" << dendl;
-          return -EINVAL;
-        }
-      } else {
-        I *img_ctx = I::create("", img_id, nullptr, io_ctx, false);
-        r = img_ctx->state->open(0);
-        if (r < 0) {
-          lderr(cct) << "error opening image id "<< img_id << ": "
-                     << cpp_strerror(r) << dendl;
-          return r;
-        }
+      std::set<std::string> image_ids;
+      r = list_mirror_images(io_ctx, image_ids);
+      if (r < 0) {
+        lderr(cct) << "failed listing images: " << cpp_strerror(r) << dendl;
+        return r;
+      }
 
-        r = image_disable(img_ctx, false);
-        int close_r = img_ctx->state->close();
-        if (r < 0) {
-          lderr(cct) << "error disabling mirroring for image id " << img_id
-                     << cpp_strerror(r) << dendl;
-          return r;
-        } else if (close_r < 0) {
-          lderr(cct) << "failed to close image id " << img_id << ": "
-                     << cpp_strerror(close_r) << dendl;
-          return close_r;
+      for (const auto& img_id : image_ids) {
+        if (current_mirror_mode == cls::rbd::MIRROR_MODE_IMAGE) {
+          cls::rbd::MirrorImage mirror_image;
+          r = cls_client::mirror_image_get(&io_ctx, img_id, &mirror_image);
+          if (r < 0 && r != -ENOENT) {
+            lderr(cct) << "failed to retrieve mirroring state for image id "
+                       << img_id << ": " << cpp_strerror(r) << dendl;
+            return r;
+          }
+          if (mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_ENABLED) {
+            lderr(cct) << "failed to disable mirror mode: there are still "
+                       << "images with mirroring enabled" << dendl;
+            return -EINVAL;
+          }
+        } else {
+          I *img_ctx = I::create("", img_id, nullptr, io_ctx, false);
+          r = img_ctx->state->open(0);
+          if (r < 0) {
+            lderr(cct) << "error opening image id "<< img_id << ": "
+                       << cpp_strerror(r) << dendl;
+            return r;
+          }
+
+          r = image_disable(img_ctx, false);
+          int close_r = img_ctx->state->close();
+          if (r == -EBUSY) {
+            pending_busy = true;
+          } else if (r < 0) {
+            lderr(cct) << "error disabling mirroring for image id " << img_id
+                       << cpp_strerror(r) << dendl;
+            return r;
+          } else if (close_r < 0) {
+            lderr(cct) << "failed to close image id " << img_id << ": "
+                       << cpp_strerror(close_r) << dendl;
+            return close_r;
+          } else if (pending_busy) {
+            // at least one mirrored image was successfully disabled, so we can
+            // retry any failures caused by busy parent/child relationships
+            retry_busy = true;
+          }
         }
       }
+
+      if (!retry_busy && pending_busy) {
+        lderr(cct) << "error disabling mirroring for one or more images"
+                   << dendl;
+        return -EBUSY;
+      } else if (!retry_busy) {
+        break;
+      }
     }
   }
 
-- 
2.39.5