]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
librbd: object cacher should re-use read state machine
authorJason Dillaman <dillaman@redhat.com>
Tue, 7 Nov 2017 19:36:10 +0000 (14:36 -0500)
committerJason Dillaman <dillaman@redhat.com>
Thu, 16 Nov 2017 12:31:59 +0000 (07:31 -0500)
This adds support for sparse-reads and ensures all object reads
utilize a single, tested code path.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
src/librbd/LibrbdWriteback.cc
src/librbd/io/ImageRequest.cc
src/librbd/io/ObjectRequest.cc
src/librbd/io/ObjectRequest.h
src/librbd/io/ReadResult.cc
src/librbd/io/ReadResult.h
src/test/librbd/io/test_mock_ObjectRequest.cc

index f0be832eb171f45862285a0371afb78107dee66e..7bc90cadefb856075b4583004c67aa6244cf2668 100644 (file)
@@ -20,6 +20,7 @@
 #include "librbd/Utils.h"
 #include "librbd/io/AioCompletion.h"
 #include "librbd/io/ObjectRequest.h"
+#include "librbd/io/ReadResult.h"
 
 #include "include/assert.h"
 
@@ -204,30 +205,29 @@ namespace librbd {
                              const ZTracer::Trace &parent_trace,
                              Context *onfinish)
   {
-    // on completion, take the mutex and then call onfinish.
-    Context *req = new C_ReadRequest(m_ictx->cct, onfinish, &m_lock);
-
-    {
-      RWLock::RLocker snap_locker(m_ictx->snap_lock);
-      if (m_ictx->object_map != nullptr &&
-          !m_ictx->object_map->object_may_exist(object_no)) {
-        m_ictx->op_work_queue->queue(req, -ENOENT);
-       return;
-      }
+    ZTracer::Trace trace;
+    if (parent_trace.valid()) {
+      trace.init("", &m_ictx->trace_endpoint, &parent_trace);
+      trace.copy_name("cache read " + oid.name);
+      trace.event("start");
     }
 
-    librados::ObjectReadOperation op;
-    op.read(off, len, pbl, NULL);
-    op.set_op_flags2(op_flags);
-    int flags = m_ictx->get_read_flags(snapid);
-
-    librados::AioCompletion *rados_completion =
-      util::create_rados_callback(req);
-    int r = m_ictx->data_ctx.aio_operate(
-      oid.name, rados_completion, &op, flags, nullptr,
-      (parent_trace.valid() ? parent_trace.get_info() : nullptr));
-    rados_completion->release();
-    assert(r >= 0);
+    // on completion, take the mutex and then call onfinish.
+    onfinish = new C_ReadRequest(m_ictx->cct, onfinish, &m_lock);
+
+    // re-use standard object read state machine
+    auto aio_comp = io::AioCompletion::create_and_start(onfinish, m_ictx,
+                                                        io::AIO_TYPE_READ);
+    aio_comp->read_result = io::ReadResult{pbl};
+    aio_comp->set_request_count(1);
+
+    auto req_comp = new io::ReadResult::C_SparseReadRequest<>(
+      aio_comp, {{0, len}}, false);
+    auto req = io::ObjectReadRequest<>::create(m_ictx, oid.name, object_no, off,
+                                               len, snapid, op_flags, true,
+                                               trace, req_comp);
+    req_comp->request = req;
+    req->send();
   }
 
   bool LibrbdWriteback::may_copy_on_write(const object_t& oid, uint64_t read_off, uint64_t read_len, snapid_t snapid)
index 9c2b1989e18845e0d817674c1086aede8ef38a9c..50a9f30ee448b70de994229ac12a00476e5fb393 100644 (file)
@@ -325,10 +325,10 @@ void ImageReadRequest<I>::send_request() {
                      << dendl;
 
       auto req_comp = new io::ReadResult::C_SparseReadRequest<I>(
-        aio_comp, std::move(extent.buffer_extents));
+        aio_comp, std::move(extent.buffer_extents), true);
       ObjectReadRequest<I> *req = ObjectReadRequest<I>::create(
         &image_ctx, extent.oid.name, extent.objectno, extent.offset,
-        extent.length, snap_id, m_op_flags, this->m_trace, req_comp);
+        extent.length, snap_id, m_op_flags, false, this->m_trace, req_comp);
       req_comp->request = req;
       req->send();
     }
index 608581b87f3bea085402ce36a1e04a12ad43793e..12c7afd85b0119265c195c31b0a9575fd2624c58 100644 (file)
@@ -215,12 +215,12 @@ template <typename I>
 ObjectReadRequest<I>::ObjectReadRequest(I *ictx, const std::string &oid,
                                         uint64_t objectno, uint64_t offset,
                                         uint64_t len, librados::snap_t snap_id,
-                                        int op_flags,
+                                        int op_flags, bool cache_initiated,
                                         const ZTracer::Trace &parent_trace,
                                         Context *completion)
   : ObjectRequest<I>(ictx, oid, objectno, offset, len, snap_id, false, "read",
                      parent_trace, completion),
-    m_op_flags(op_flags) {
+    m_op_flags(op_flags), m_cache_initiated(cache_initiated) {
 }
 
 template <typename I>
@@ -228,7 +228,7 @@ void ObjectReadRequest<I>::send() {
   I *image_ctx = this->m_ictx;
   ldout(image_ctx->cct, 20) << dendl;
 
-  if (image_ctx->object_cacher != nullptr) {
+  if (!m_cache_initiated && image_ctx->object_cacher != nullptr) {
     read_cache();
   } else {
     read_object();
@@ -325,6 +325,11 @@ void ObjectReadRequest<I>::handle_read_object(int r) {
 template <typename I>
 void ObjectReadRequest<I>::read_parent() {
   I *image_ctx = this->m_ictx;
+  if (m_cache_initiated) {
+    this->finish(-ENOENT);
+    return;
+  }
+
   uint64_t object_overlap = 0;
   Extents parent_extents;
   {
index 899196beb72187304d68d98754d0a86ee0aaf8fd..93a373c40df67e8003eb268d0180fa9f2cbd1daa 100644 (file)
@@ -152,17 +152,18 @@ public:
   static ObjectReadRequest* create(ImageCtxT *ictx, const std::string &oid,
                                    uint64_t objectno, uint64_t offset,
                                    uint64_t len, librados::snap_t snap_id,
-                                   int op_flags,
+                                   int op_flags, bool cache_initiated,
                                    const ZTracer::Trace &parent_trace,
                                    Context *completion) {
     return new ObjectReadRequest(ictx, oid, objectno, offset, len,
-                                 snap_id, op_flags, parent_trace, completion);
+                                 snap_id, op_flags, cache_initiated,
+                                 parent_trace, completion);
   }
 
   ObjectReadRequest(ImageCtxT *ictx, const std::string &oid,
                     uint64_t objectno, uint64_t offset, uint64_t len,
-                    librados::snap_t snap_id,
-                    int op_flags, const ZTracer::Trace &parent_trace,
+                    librados::snap_t snap_id, int op_flags,
+                    bool cache_initiated, const ZTracer::Trace &parent_trace,
                     Context *completion);
 
   bool should_complete(int r) override;
@@ -217,6 +218,7 @@ private:
    */
 
   int m_op_flags;
+  bool m_cache_initiated;
 
   ceph::bufferlist m_read_data;
   ExtentMap m_ext_map;
index 448de54200a167a3b2277068e3cddc3b3c8bcc6b..3b126871d1407bd2ea27f27114de67978cf46925 100644 (file)
@@ -117,7 +117,10 @@ void ReadResult::C_SparseReadRequestBase::finish(ExtentMap &extent_map,
   ldout(cct, 10) << "C_SparseReadRequestBase: r = " << r
                  << dendl;
 
-  if (r >= 0 || r == -ENOENT) {
+  if (ignore_enoent && r == -ENOENT) {
+    r = 0;
+  }
+  if (r >= 0) {
     ldout(cct, 10) << " got " << extent_map
                    << " for " << buffer_extents
                    << " bl " << bl.length() << dendl;
index f11e83286a6f4200c55e2bfc66d76d49ea0f0301..ab0e4dbe4f182189a7eb0ce5ff52453c35e7f9ee 100644 (file)
@@ -48,8 +48,10 @@ public:
   };
 
   struct C_SparseReadRequestBase : public C_ReadRequest {
-    C_SparseReadRequestBase(AioCompletion *aio_completion)
-      : C_ReadRequest(aio_completion) {
+    bool ignore_enoent;
+
+    C_SparseReadRequestBase(AioCompletion *aio_completion, bool ignore_enoent)
+      : C_ReadRequest(aio_completion), ignore_enoent(ignore_enoent) {
     }
 
     using C_ReadRequest::finish;
@@ -62,8 +64,9 @@ public:
     ObjectReadRequest<ImageCtxT> *request;
     Extents buffer_extents;
 
-    C_SparseReadRequest(AioCompletion *aio_completion, Extents&& buffer_extents)
-      : C_SparseReadRequestBase(aio_completion),
+    C_SparseReadRequest(AioCompletion *aio_completion, Extents&& buffer_extents,
+                        bool ignore_enoent)
+      : C_SparseReadRequestBase(aio_completion, ignore_enoent),
         buffer_extents(std::move(buffer_extents)) {
     }
 
index 3f0b4f571325482031550343ce365d0d412b40ff..7e6eb15d1a8a25d88da2c4ed443e72fff53b975a 100644 (file)
@@ -214,7 +214,7 @@ TEST_F(TestMockIoObjectRequest, Read) {
 
   C_SaferCond ctx;
   auto req = MockObjectReadRequest::create(
-    &mock_image_ctx, "object0", 0, 0, 4096, CEPH_NOSNAP, 0, {}, &ctx);
+    &mock_image_ctx, "object0", 0, 0, 4096, CEPH_NOSNAP, 0, false, {}, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
 }
@@ -242,7 +242,7 @@ TEST_F(TestMockIoObjectRequest, SparseReadThreshold) {
   C_SaferCond ctx;
   auto req = MockObjectReadRequest::create(
     &mock_image_ctx, "object0", 0, 0, ictx->sparse_read_threshold_bytes,
-    CEPH_NOSNAP, 0, {}, &ctx);
+    CEPH_NOSNAP, 0, false, {}, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
 }
@@ -267,7 +267,7 @@ TEST_F(TestMockIoObjectRequest, ReadError) {
 
   C_SaferCond ctx;
   auto req = MockObjectReadRequest::create(
-    &mock_image_ctx, "object0", 0, 0, 4096, CEPH_NOSNAP, 0, {}, &ctx);
+    &mock_image_ctx, "object0", 0, 0, 4096, CEPH_NOSNAP, 0, false, {}, &ctx);
   req->send();
   ASSERT_EQ(-EPERM, ctx.wait());
 }
@@ -291,7 +291,7 @@ TEST_F(TestMockIoObjectRequest, CacheRead) {
 
   C_SaferCond ctx;
   auto req = MockObjectReadRequest::create(
-    &mock_image_ctx, "object0", 0, 0, 4096, CEPH_NOSNAP, 0, {}, &ctx);
+    &mock_image_ctx, "object0", 0, 0, 4096, CEPH_NOSNAP, 0, false, {}, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
 }
@@ -315,7 +315,7 @@ TEST_F(TestMockIoObjectRequest, CacheReadError) {
 
   C_SaferCond ctx;
   auto req = MockObjectReadRequest::create(
-    &mock_image_ctx, "object0", 0, 0, 4096, CEPH_NOSNAP, 0, {}, &ctx);
+    &mock_image_ctx, "object0", 0, 0, 4096, CEPH_NOSNAP, 0, false, {}, &ctx);
   req->send();
   ASSERT_EQ(-EPERM, ctx.wait());
 }
@@ -361,7 +361,7 @@ TEST_F(TestMockIoObjectRequest, ParentRead) {
 
   C_SaferCond ctx;
   auto req = MockObjectReadRequest::create(
-    &mock_image_ctx, "object0", 0, 0, 4096, CEPH_NOSNAP, 0, {}, &ctx);
+    &mock_image_ctx, "object0", 0, 0, 4096, CEPH_NOSNAP, 0, false, {}, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
 }
@@ -407,11 +407,51 @@ TEST_F(TestMockIoObjectRequest, ParentReadError) {
 
   C_SaferCond ctx;
   auto req = MockObjectReadRequest::create(
-    &mock_image_ctx, "object0", 0, 0, 4096, CEPH_NOSNAP, 0, {}, &ctx);
+    &mock_image_ctx, "object0", 0, 0, 4096, CEPH_NOSNAP, 0, false, {}, &ctx);
   req->send();
   ASSERT_EQ(-EPERM, ctx.wait());
 }
 
+TEST_F(TestMockIoObjectRequest, CacheInitiated) {
+  REQUIRE_FEATURE(RBD_FEATURE_LAYERING);
+
+  librbd::Image image;
+  librbd::RBD rbd;
+  ASSERT_EQ(0, rbd.open(m_ioctx, image, m_image_name.c_str(), NULL));
+  ASSERT_EQ(0, image.snap_create("one"));
+  ASSERT_EQ(0, image.snap_protect("one"));
+  image.close();
+
+  std::string clone_name = get_temp_image_name();
+  int order = 0;
+  ASSERT_EQ(0, rbd.clone(m_ioctx, m_image_name.c_str(), "one", m_ioctx,
+                         clone_name.c_str(), RBD_FEATURE_LAYERING, &order));
+
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(clone_name, &ictx));
+  ictx->sparse_read_threshold_bytes = 8096;
+  ictx->clone_copy_on_read = false;
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+  mock_image_ctx.parent = &mock_image_ctx;
+
+  MockObjectMap mock_object_map;
+  if (ictx->test_features(RBD_FEATURE_OBJECT_MAP)) {
+    mock_image_ctx.object_map = &mock_object_map;
+  }
+
+  InSequence seq;
+  expect_object_may_exist(mock_image_ctx, 0, true);
+  expect_get_read_flags(mock_image_ctx, CEPH_NOSNAP, 0);
+  expect_read(mock_image_ctx, "object0", 0, 4096, "", -ENOENT);
+
+  C_SaferCond ctx;
+  auto req = MockObjectReadRequest::create(
+    &mock_image_ctx, "object0", 0, 0, 4096, CEPH_NOSNAP, 0, true, {}, &ctx);
+  req->send();
+  ASSERT_EQ(-ENOENT, ctx.wait());
+}
+
 TEST_F(TestMockIoObjectRequest, CopyOnRead) {
   REQUIRE_FEATURE(RBD_FEATURE_LAYERING);
 
@@ -458,7 +498,7 @@ TEST_F(TestMockIoObjectRequest, CopyOnRead) {
 
   C_SaferCond ctx;
   auto req = MockObjectReadRequest::create(
-    &mock_image_ctx, "object0", 0, 0, 4096, CEPH_NOSNAP, 0, {}, &ctx);
+    &mock_image_ctx, "object0", 0, 0, 4096, CEPH_NOSNAP, 0, false, {}, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
 }