ceph_test_rados_api_*: wait for snap trim on ENOENT during cleanup

Assume that an ENOENT removing an object during cleanup means the
head is a whiteout because there are clones.  When that happens
just sleep for a bit and then retry in the hopes that snap
trimming has happened.

Time out (and fail) after retrying for 10 minutes' worth of tries.

Fixes: http://tracker.ceph.com/issues/19948
Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/test/librados/TestCase.cc b/src/test/librados/TestCase.cc
index 9df0cce9f5317dd0e16f49e11df714b6a18b7b93..37dbd2a923bfc8d7f07a286bdde3e630e9dcb796 100644 (file)
--- a/src/test/librados/TestCase.cc
+++ b/src/test/librados/TestCase.cc
@@ -368,19 +368,38 @@ void RadosTestPP::cleanup_namespace(librados::IoCtx ioctx, std::string ns)
 {
   ioctx.snap_set_read(librados::SNAP_HEAD);
   ioctx.set_namespace(ns);
-  for (NObjectIterator it = ioctx.nobjects_begin();
-       it != ioctx.nobjects_end(); ++it) {
-    ioctx.locator_set_key(it->get_locator());
-    ObjectWriteOperation op;
-    op.remove();
-
-    librados::AioCompletion *completion = s_cluster.aio_create_completion();
-    auto sg = make_scope_guard([&] { completion->release(); });
-
-    ASSERT_EQ(0, ioctx.aio_operate(it->get_oid(), completion, &op,
-                                  librados::OPERATION_IGNORE_CACHE));
-    completion->wait_for_safe();
-    ASSERT_EQ(0, completion->get_return_value());
+  int tries = 600;
+  while (--tries) {
+    int got_enoent = 0;
+    for (NObjectIterator it = ioctx.nobjects_begin();
+        it != ioctx.nobjects_end(); ++it) {
+      ioctx.locator_set_key(it->get_locator());
+      ObjectWriteOperation op;
+      op.remove();
+      librados::AioCompletion *completion = s_cluster.aio_create_completion();
+      auto sg = make_scope_guard([&] { completion->release(); });
+      ASSERT_EQ(0, ioctx.aio_operate(it->get_oid(), completion, &op,
+                                    librados::OPERATION_IGNORE_CACHE));
+      completion->wait_for_safe();
+      if (completion->get_return_value() == -ENOENT) {
+       ++got_enoent;
+       std::cout << " got ENOENT removing " << it->get_oid() << std::endl;
+      } else {
+       ASSERT_EQ(0, completion->get_return_value());
+      }
+    }
+    if (!got_enoent) {
+      break;
+    }
+    std::cout << " got ENOENT on " << got_enoent
+             << " objects, waiting a bit for snap"
+             << " trimming before retrying " << tries << " more times..."
+             << std::endl;
+    sleep(1);
+  }
+  if (tries == 0) {
+    std::cout << "failed to clean up" << std::endl;
+    ASSERT_TRUE(false);
   }
 }