osd/ReplicatedPG: make cache-[try-]flush CACHE instead of WR ops

This will allow us to send a flush op on a snap.

Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/src/include/rados.h b/src/include/rados.h
index 894856649c464a1283b774eff6bbc770700c1f89..49a8b9dc5155f6825a288ed8b204e158f0f1daab 100644 (file)
--- a/src/include/rados.h
+++ b/src/include/rados.h
@@ -228,9 +228,9 @@ enum {
        CEPH_OSD_OP_UNDIRTY   = CEPH_OSD_OP_MODE_WR | CEPH_OSD_OP_TYPE_DATA | 28,
        CEPH_OSD_OP_ISDIRTY   = CEPH_OSD_OP_MODE_RD | CEPH_OSD_OP_TYPE_DATA | 29,
        CEPH_OSD_OP_COPY_GET = CEPH_OSD_OP_MODE_RD | CEPH_OSD_OP_TYPE_DATA | 30,
-       CEPH_OSD_OP_CACHE_FLUSH = CEPH_OSD_OP_MODE_WR | CEPH_OSD_OP_TYPE_DATA | 31,
+       CEPH_OSD_OP_CACHE_FLUSH = CEPH_OSD_OP_MODE_CACHE | CEPH_OSD_OP_TYPE_DATA | 31,
        CEPH_OSD_OP_CACHE_EVICT = CEPH_OSD_OP_MODE_CACHE | CEPH_OSD_OP_TYPE_DATA | 32,
-       CEPH_OSD_OP_CACHE_TRY_FLUSH = CEPH_OSD_OP_MODE_WR | CEPH_OSD_OP_TYPE_DATA | 33,
+       CEPH_OSD_OP_CACHE_TRY_FLUSH = CEPH_OSD_OP_MODE_CACHE | CEPH_OSD_OP_TYPE_DATA | 33,
 
        /* convert tmap to omap */
        CEPH_OSD_OP_TMAP2OMAP = CEPH_OSD_OP_MODE_RMW | CEPH_OSD_OP_TYPE_DATA | 34,

diff --git a/src/include/rados/librados.hpp b/src/include/rados/librados.hpp
index 411c90ff2fc9866ed1d461fda792654145ffcca5..009c81baca93e5537d59e9ccd4d05bc2c516c238 100644 (file)
--- a/src/include/rados/librados.hpp
+++ b/src/include/rados/librados.hpp
@@ -310,24 +310,6 @@ namespace librados
      */
     void undirty();
 
-    /**
-     * flush a cache tier object to backing tier; will block racing
-     * updates.
-     *
-     * This should be used in concert with OPERATION_IGNORE_CACHE to avoid
-     * triggering a promotion.
-     */
-    void cache_flush();
-
-    /**
-     * Flush a cache tier object to backing tier; will EAGAIN if we race
-     * with an update.  Must be used with the SKIPRWLOCKS flag.
-     *
-     * This should be used in concert with OPERATION_IGNORE_CACHE to avoid
-     * triggering a promotion.
-     */
-    void cache_try_flush();
-
     friend class IoCtx;
   };
 
@@ -454,6 +436,24 @@ namespace librados
      */
     void is_dirty(bool *isdirty, int *prval);
 
+    /**
+     * flush a cache tier object to backing tier; will block racing
+     * updates.
+     *
+     * This should be used in concert with OPERATION_IGNORE_CACHE to avoid
+     * triggering a promotion.
+     */
+    void cache_flush();
+
+    /**
+     * Flush a cache tier object to backing tier; will EAGAIN if we race
+     * with an update.  Must be used with the SKIPRWLOCKS flag.
+     *
+     * This should be used in concert with OPERATION_IGNORE_CACHE to avoid
+     * triggering a promotion.
+     */
+    void cache_try_flush();
+
     /**
      * evict a clean cache tier object
      *

diff --git a/src/librados/librados.cc b/src/librados/librados.cc
index 70686be7a930a0a79cfd031b939f2b96988ece5a..278ddb679d7b26fec386022ceff059f829a06400 100644 (file)
--- a/src/librados/librados.cc
+++ b/src/librados/librados.cc
@@ -403,13 +403,13 @@ void librados::ObjectWriteOperation::undirty()
   o->undirty();
 }
 
-void librados::ObjectWriteOperation::cache_flush()
+void librados::ObjectReadOperation::cache_flush()
 {
   ::ObjectOperation *o = (::ObjectOperation *)impl;
   o->cache_flush();
 }
 
-void librados::ObjectWriteOperation::cache_try_flush()
+void librados::ObjectReadOperation::cache_try_flush()
 {
   ::ObjectOperation *o = (::ObjectOperation *)impl;
   o->cache_try_flush();

diff --git a/src/test/librados/tier.cc b/src/test/librados/tier.cc
index de8323f6855f9a0a33b32bec0a6832c658e02aa2..1a714c5bddcb1d33e5af1ad5b60753ba0f14aed3 100644 (file)
--- a/src/test/librados/tier.cc
+++ b/src/test/librados/tier.cc
@@ -952,13 +952,13 @@ TEST(LibRadosTier, TryFlush) {
 
   // flush
   {
-    ObjectWriteOperation op;
+    ObjectReadOperation op;
     op.cache_try_flush();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_OVERLAY |
-      librados::OPERATION_SKIPRWLOCKS));
+      librados::OPERATION_SKIPRWLOCKS, NULL));
     completion->wait_for_safe();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
@@ -1088,12 +1088,12 @@ TEST(LibRadosTier, Flush) {
 
   // flush
   {
-    ObjectWriteOperation op;
+    ObjectReadOperation op;
     op.cache_flush();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
-      librados::OPERATION_IGNORE_OVERLAY));
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
     completion->wait_for_safe();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
@@ -1153,12 +1153,12 @@ TEST(LibRadosTier, Flush) {
 
   // flush whiteout
   {
-    ObjectWriteOperation op;
+    ObjectReadOperation op;
     op.cache_flush();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
-      librados::OPERATION_IGNORE_OVERLAY));
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
     completion->wait_for_safe();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
@@ -1244,12 +1244,12 @@ TEST(LibRadosTier, FlushWriteRaces) {
 
   // flush + write
   {
-    ObjectWriteOperation op;
+    ObjectReadOperation op;
     op.cache_flush();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
-      librados::OPERATION_IGNORE_OVERLAY));
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
 
     ObjectWriteOperation op2;
     op2.write_full(bl);
@@ -1278,13 +1278,13 @@ TEST(LibRadosTier, FlushWriteRaces) {
 
     // try-flush + write
     {
-      ObjectWriteOperation op;
+      ObjectReadOperation op;
       op.cache_try_flush();
       librados::AioCompletion *completion = cluster.aio_create_completion();
       ASSERT_EQ(0, cache_ioctx.aio_operate(
         "foo", completion, &op,
        librados::OPERATION_IGNORE_OVERLAY |
-       librados::OPERATION_SKIPRWLOCKS));
+       librados::OPERATION_SKIPRWLOCKS, NULL));
 
       ObjectWriteOperation op2;
       op2.write_full(bl);
@@ -1362,19 +1362,19 @@ TEST(LibRadosTier, FlushTryFlushRaces) {
 
   // flush + flush
   {
-    ObjectWriteOperation op;
+    ObjectReadOperation op;
     op.cache_flush();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
-      librados::OPERATION_IGNORE_OVERLAY));
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
 
-    ObjectWriteOperation op2;
+    ObjectReadOperation op2;
     op2.cache_flush();
     librados::AioCompletion *completion2 = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion2, &op2,
-      librados::OPERATION_IGNORE_OVERLAY));
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
 
     completion->wait_for_safe();
     completion2->wait_for_safe();
@@ -1395,20 +1395,20 @@ TEST(LibRadosTier, FlushTryFlushRaces) {
 
   // flush + try-flush
   {
-    ObjectWriteOperation op;
+    ObjectReadOperation op;
     op.cache_flush();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
-      librados::OPERATION_IGNORE_OVERLAY));
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
 
-    ObjectWriteOperation op2;
+    ObjectReadOperation op2;
     op2.cache_try_flush();
     librados::AioCompletion *completion2 = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion2, &op2,
       librados::OPERATION_IGNORE_OVERLAY |
-      librados::OPERATION_SKIPRWLOCKS));
+      librados::OPERATION_SKIPRWLOCKS, NULL));
 
     completion->wait_for_safe();
     completion2->wait_for_safe();
@@ -1432,20 +1432,20 @@ TEST(LibRadosTier, FlushTryFlushRaces) {
     // try-flush + flush
     //  (flush will not piggyback on try-flush)
     {
-      ObjectWriteOperation op;
+      ObjectReadOperation op;
       op.cache_try_flush();
       librados::AioCompletion *completion = cluster.aio_create_completion();
       ASSERT_EQ(0, cache_ioctx.aio_operate(
         "foo", completion, &op,
        librados::OPERATION_IGNORE_OVERLAY |
-       librados::OPERATION_SKIPRWLOCKS));
+       librados::OPERATION_SKIPRWLOCKS, NULL));
 
-      ObjectWriteOperation op2;
+      ObjectReadOperation op2;
       op2.cache_flush();
       librados::AioCompletion *completion2 = cluster.aio_create_completion();
       ASSERT_EQ(0, cache_ioctx.aio_operate(
         "foo", completion2, &op2,
-       librados::OPERATION_IGNORE_OVERLAY));
+       librados::OPERATION_IGNORE_OVERLAY, NULL));
 
       completion->wait_for_safe();
       completion2->wait_for_safe();
@@ -1472,21 +1472,21 @@ TEST(LibRadosTier, FlushTryFlushRaces) {
 
   // try-flush + try-flush
   {
-    ObjectWriteOperation op;
+    ObjectReadOperation op;
     op.cache_try_flush();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_OVERLAY |
-      librados::OPERATION_SKIPRWLOCKS));
+      librados::OPERATION_SKIPRWLOCKS, NULL));
 
-    ObjectWriteOperation op2;
+    ObjectReadOperation op2;
     op2.cache_try_flush();
     librados::AioCompletion *completion2 = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion2, &op2,
       librados::OPERATION_IGNORE_OVERLAY |
-      librados::OPERATION_SKIPRWLOCKS));
+      librados::OPERATION_SKIPRWLOCKS, NULL));
 
     completion->wait_for_safe();
     completion2->wait_for_safe();
@@ -1598,13 +1598,13 @@ TEST(LibRadosTier, TryFlushReadRace) {
   test_lock.Unlock();
 
   // try-flush
-  ObjectWriteOperation op;
+  ObjectReadOperation op;
   op.cache_try_flush();
   librados::AioCompletion *completion = cluster.aio_create_completion();
   ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_OVERLAY |
-      librados::OPERATION_SKIPRWLOCKS));
+      librados::OPERATION_SKIPRWLOCKS, NULL));
 
   completion->wait_for_safe();
   ASSERT_EQ(0, completion->get_return_value());

diff --git a/src/test/osd/RadosModel.h b/src/test/osd/RadosModel.h
index 22090a8cbbf9f7428671acfa50480d409e4d9b9d..01a48a1fda0ece8aac192a3617dfe5591eb16a9d 100644 (file)
--- a/src/test/osd/RadosModel.h
+++ b/src/test/osd/RadosModel.h
@@ -1761,7 +1761,7 @@ public:
 class CacheFlushOp : public TestOp {
 public:
   librados::AioCompletion *completion;
-  librados::ObjectWriteOperation op;
+  librados::ObjectReadOperation op;
   string oid;
   bool blocking;
 
@@ -1797,7 +1797,7 @@ public:
       flags = librados::OPERATION_SKIPRWLOCKS;
     }
     int r = context->io_ctx.aio_operate(context->prefix+oid, completion,
-                                       &op, flags);
+                                       &op, flags, NULL);
     assert(!r);
   }
 

diff --git a/src/tools/rados/rados.cc b/src/tools/rados/rados.cc
index 2b132faa0cabd33a0f8af241244d86aa0dc32f09..e5ee834bdea490ea22519d5ad5cdf909ea70cdb2 100644 (file)
--- a/src/tools/rados/rados.cc
+++ b/src/tools/rados/rados.cc
@@ -1051,13 +1051,14 @@ static int do_lock_cmd(std::vector<const char*> &nargs,
 
 static int do_cache_flush(IoCtx& io_ctx, string oid)
 {
-  ObjectWriteOperation op;
+  ObjectReadOperation op;
   op.cache_flush();
   librados::AioCompletion *completion =
     librados::Rados::aio_create_completion();
   io_ctx.aio_operate(oid.c_str(), completion, &op,
                     librados::OPERATION_IGNORE_CACHE |
-                    librados::OPERATION_IGNORE_OVERLAY);
+                    librados::OPERATION_IGNORE_OVERLAY,
+                    NULL);
   completion->wait_for_safe();
   int r = completion->get_return_value();
   completion->release();
@@ -1066,14 +1067,15 @@ static int do_cache_flush(IoCtx& io_ctx, string oid)
 
 static int do_cache_try_flush(IoCtx& io_ctx, string oid)
 {
-  ObjectWriteOperation op;
+  ObjectReadOperation op;
   op.cache_try_flush();
   librados::AioCompletion *completion =
     librados::Rados::aio_create_completion();
   io_ctx.aio_operate(oid.c_str(), completion, &op,
                     librados::OPERATION_IGNORE_CACHE |
                     librados::OPERATION_IGNORE_OVERLAY |
-                    librados::OPERATION_SKIPRWLOCKS);
+                    librados::OPERATION_SKIPRWLOCKS,
+                    NULL);
   completion->wait_for_safe();
   int r = completion->get_return_value();
   completion->release();