ceph_test_rados_api_tier: retry EBUSY race checks

...or else these will occasionally fail against a thrashing cluster.

Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/src/test/librados/tier.cc b/src/test/librados/tier.cc
index 883234f0ee5295d8c5ce507bc8e086098c2d40b4..6a2d0809347500feb0794baf5c33a35fbde7cc12 100644 (file)
--- a/src/test/librados/tier.cc
+++ b/src/test/librados/tier.cc
@@ -839,38 +839,45 @@ TEST(LibRadosTier, FlushWriteRaces) {
     completion2->release();
   }
 
-  // create/dirty object
-  {
-    bufferlist bl;
-    bl.append("hi there");
-    ObjectWriteOperation op;
-    op.write_full(bl);
-    ASSERT_EQ(0, base_ioctx.operate("foo", &op));
-  }
-
-  // try-flush + write
-  {
-    ObjectWriteOperation op;
-    op.cache_try_flush();
-    librados::AioCompletion *completion = cluster.aio_create_completion();
-    ASSERT_EQ(0, cache_ioctx.aio_operate(
-      "foo", completion, &op,
-      librados::OPERATION_IGNORE_OVERLAY |
-      librados::OPERATION_SKIPRWLOCKS));
-
-    ObjectWriteOperation op2;
-    op2.write_full(bl);
-    librados::AioCompletion *completion2 = cluster.aio_create_completion();
-    ASSERT_EQ(0, base_ioctx.aio_operate(
-      "foo", completion2, &op2, 0));
+  int tries = 10;
+  do {
+    // create/dirty object
+    {
+      bufferlist bl;
+      bl.append("hi there");
+      ObjectWriteOperation op;
+      op.write_full(bl);
+      ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+    }
 
-    completion->wait_for_safe();
-    completion2->wait_for_safe();
-    ASSERT_EQ(-EBUSY, completion->get_return_value());
-    ASSERT_EQ(0, completion2->get_return_value());
-    completion->release();
-    completion2->release();
-  }
+    // try-flush + write
+    {
+      ObjectWriteOperation op;
+      op.cache_try_flush();
+      librados::AioCompletion *completion = cluster.aio_create_completion();
+      ASSERT_EQ(0, cache_ioctx.aio_operate(
+        "foo", completion, &op,
+       librados::OPERATION_IGNORE_OVERLAY |
+       librados::OPERATION_SKIPRWLOCKS));
+
+      ObjectWriteOperation op2;
+      op2.write_full(bl);
+      librados::AioCompletion *completion2 = cluster.aio_create_completion();
+      ASSERT_EQ(0, base_ioctx.aio_operate("foo", completion2, &op2, 0));
+
+      completion->wait_for_safe();
+      completion2->wait_for_safe();
+      int r = completion->get_return_value();
+      ASSERT_TRUE(r == -EBUSY || r == 0);
+      ASSERT_EQ(0, completion2->get_return_value());
+      completion->release();
+      completion2->release();
+      if (r == -EBUSY)
+       break;
+      cout << "didn't get EBUSY, trying again" << std::endl;
+    }
+    ASSERT_TRUE(--tries);
+  } while (true);
 
   // tear down tiers
   ASSERT_EQ(0, cluster.mon_command(
@@ -986,39 +993,47 @@ TEST(LibRadosTier, FlushTryFlushRaces) {
   }
 
   // create/dirty object
-  {
-    bufferlist bl;
-    bl.append("hi there");
-    ObjectWriteOperation op;
-    op.write_full(bl);
-    ASSERT_EQ(0, base_ioctx.operate("foo", &op));
-  }
-
-  // try-flush + flush
-  //  (flush will not piggyback on try-flush)
-  {
-    ObjectWriteOperation op;
-    op.cache_try_flush();
-    librados::AioCompletion *completion = cluster.aio_create_completion();
-    ASSERT_EQ(0, cache_ioctx.aio_operate(
-      "foo", completion, &op,
-      librados::OPERATION_IGNORE_OVERLAY |
-      librados::OPERATION_SKIPRWLOCKS));
-
-    ObjectWriteOperation op2;
-    op2.cache_flush();
-    librados::AioCompletion *completion2 = cluster.aio_create_completion();
-    ASSERT_EQ(0, cache_ioctx.aio_operate(
-      "foo", completion2, &op2,
-      librados::OPERATION_IGNORE_OVERLAY));
+  int tries = 10;
+  do {
+    {
+      bufferlist bl;
+      bl.append("hi there");
+      ObjectWriteOperation op;
+      op.write_full(bl);
+      ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+    }
 
-    completion->wait_for_safe();
-    completion2->wait_for_safe();
-    ASSERT_EQ(-EBUSY, completion->get_return_value());
-    ASSERT_EQ(0, completion2->get_return_value());
-    completion->release();
-    completion2->release();
-  }
+    // try-flush + flush
+    //  (flush will not piggyback on try-flush)
+    {
+      ObjectWriteOperation op;
+      op.cache_try_flush();
+      librados::AioCompletion *completion = cluster.aio_create_completion();
+      ASSERT_EQ(0, cache_ioctx.aio_operate(
+        "foo", completion, &op,
+       librados::OPERATION_IGNORE_OVERLAY |
+       librados::OPERATION_SKIPRWLOCKS));
+
+      ObjectWriteOperation op2;
+      op2.cache_flush();
+      librados::AioCompletion *completion2 = cluster.aio_create_completion();
+      ASSERT_EQ(0, cache_ioctx.aio_operate(
+        "foo", completion2, &op2,
+       librados::OPERATION_IGNORE_OVERLAY));
+
+      completion->wait_for_safe();
+      completion2->wait_for_safe();
+      int r = completion->get_return_value();
+      ASSERT_TRUE(r == -EBUSY || r == 0);
+      ASSERT_EQ(0, completion2->get_return_value());
+      completion->release();
+      completion2->release();
+      if (r == -EBUSY)
+       break;
+      cout << "didn't get EBUSY, trying again" << std::endl;
+    }
+    ASSERT_TRUE(--tries);
+  } while (true);
 
   // create/dirty object
   {