From: Haomai Wang <haomaiwang@gmail.com>
Date: Tue, 4 Aug 2015 09:22:40 +0000 (+0800)
Subject: librbd: Add event notify interfaces
X-Git-Tag: v10.0.2~142^2~16
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=c3a1edbd2aaaca3c66204b1e1435045cdb4294ef;p=ceph.git

librbd: Add event notify interfaces

Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
---

diff --git a/src/include/rbd/librbd.h b/src/include/rbd/librbd.h
index ba64219d3be7..2f665916566c 100644
--- a/src/include/rbd/librbd.h
+++ b/src/include/rbd/librbd.h
@@ -191,6 +191,7 @@ CEPH_RBD_API int rbd_get_parent_info(rbd_image_t image,
 			             char *parent_snapname,
                                      size_t psnapnamelen);
 CEPH_RBD_API int rbd_get_flags(rbd_image_t image, uint64_t *flags);
+CEPH_RBD_API int rbd_set_image_notification(rbd_image_t image, int fd, int type);
 
 /* exclusive lock feature */
 CEPH_RBD_API int rbd_is_exclusive_lock_owner(rbd_image_t image, int *is_owner);
@@ -471,6 +472,7 @@ CEPH_RBD_API int rbd_aio_read2(rbd_image_t image, uint64_t off, size_t len,
                                char *buf, rbd_completion_t c, int op_flags);
 CEPH_RBD_API int rbd_aio_discard(rbd_image_t image, uint64_t off, uint64_t len,
                                  rbd_completion_t c);
+
 CEPH_RBD_API int rbd_aio_create_completion(void *cb_arg,
                                            rbd_callback_t complete_cb,
                                            rbd_completion_t *c);
@@ -497,6 +499,8 @@ CEPH_RBD_API int rbd_aio_flush(rbd_image_t image, rbd_completion_t c);
  */
 CEPH_RBD_API int rbd_invalidate_cache(rbd_image_t image);
 
+CEPH_RBD_API int rbd_poll_io_events(rbd_image_t image, rbd_completion_t *comps, int numcomp, int max);
+
 CEPH_RBD_API int rbd_metadata_get(rbd_image_t image, const char *key, char *value, size_t *val_len);
 CEPH_RBD_API int rbd_metadata_set(rbd_image_t image, const char *key, const char *value);
 CEPH_RBD_API int rbd_metadata_remove(rbd_image_t image, const char *key);
diff --git a/src/include/rbd/librbd.hpp b/src/include/rbd/librbd.hpp
index a64ffa6b58ca..49a6f783bf00 100644
--- a/src/include/rbd/librbd.hpp
+++ b/src/include/rbd/librbd.hpp
@@ -147,6 +147,7 @@ public:
   int update_features(uint64_t features, bool enabled);
   int overlap(uint64_t *overlap);
   int get_flags(uint64_t *flags);
+  int set_image_notification(int fd, int type);
 
   /* exclusive lock feature */
   int is_exclusive_lock_owner(bool *is_owner);
@@ -283,6 +284,8 @@ public:
    */
   int invalidate_cache();
 
+  int poll_io_events(RBD::AioCompletion **comps, int numcomp, int max);
+
   int metadata_get(const std::string &key, std::string *value);
   int metadata_set(const std::string &key, const std::string &value);
   int metadata_remove(const std::string &key);
diff --git a/src/librbd/AioCompletion.cc b/src/librbd/AioCompletion.cc
index 053df429af29..9d25c0bd61fb 100644
--- a/src/librbd/AioCompletion.cc
+++ b/src/librbd/AioCompletion.cc
@@ -98,12 +98,19 @@ namespace librbd {
       async_op.finish_op();
     }
 
+    lock.Unlock();
     if (complete_cb) {
-      lock.Unlock();
       complete_cb(rbd_comp, complete_arg);
-      lock.Lock();
     }
+
+    lock.Lock();
     done = true;
+    if (event_notify && ictx->event_socket.is_valid()) {
+      ictx->completed_reqs_lock.Lock();
+      ictx->completed_reqs.push_back(&m_xlist_item);
+      ictx->completed_reqs_lock.Unlock();
+      ictx->event_socket.notify();
+    }
     cond.Signal();
     tracepoint(librbd, aio_complete_exit);
   }
diff --git a/src/librbd/AioCompletion.h b/src/librbd/AioCompletion.h
index 532f7e261536..1ef37a130984 100644
--- a/src/librbd/AioCompletion.h
+++ b/src/librbd/AioCompletion.h
@@ -63,6 +63,8 @@ namespace librbd {
     AsyncOperation async_op;
 
     uint64_t journal_tid;
+    xlist<AioCompletion*>::item m_xlist_item;
+    bool event_notify;
 
     AioCompletion() : lock("AioCompletion::lock", true, false),
 		      done(false), rval(0), complete_cb(NULL),
@@ -72,6 +74,7 @@ namespace librbd {
 		      aio_type(AIO_TYPE_NONE),
 		      read_bl(NULL), read_buf(NULL), read_buf_len(0),
                       journal_tid(0) {
+                      m_xlist_item(this), event_notify(false) {
     }
     ~AioCompletion() {
     }
@@ -128,8 +131,12 @@ namespace librbd {
       assert(ref > 0);
       int n = --ref;
       lock.Unlock();
-      if (!n)
-	delete this;
+      if (!n) {
+        ictx->completed_reqs_lock.Lock();
+        m_xlist_item.remove_myself();
+        ictx->completed_reqs_lock.Unlock();
+        delete this;
+      }
     }
 
     void block() {
@@ -145,6 +152,11 @@ namespace librbd {
         complete(cct);
       }
     }
+
+    void enable_event_notify() {
+      Mutex::Locker l(lock);
+      event_notify = true;
+    }
   };
 
   class C_AioRequest : public Context {
diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc
index 2f0bb39c8e1b..69fc596421ff 100644
--- a/src/librbd/ImageCtx.cc
+++ b/src/librbd/ImageCtx.cc
@@ -139,6 +139,7 @@ struct C_InvalidateCache : public Context {
       object_map_lock(unique_lock_name("librbd::ImageCtx::object_map_lock", this)),
       async_ops_lock(unique_lock_name("librbd::ImageCtx::async_ops_lock", this)),
       copyup_list_lock(unique_lock_name("librbd::ImageCtx::copyup_list_lock", this)),
+      completed_reqs_lock(unique_lock_name("librbd::ImageCtx::completed_reqs_lock", this)),
       extra_read_flags(0),
       old_format(true),
       order(0), size(0), features(0),
diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h
index 28e8f7455afb..c0d4e5fe1f18 100644
--- a/src/librbd/ImageCtx.h
+++ b/src/librbd/ImageCtx.h
@@ -12,6 +12,7 @@
 #include <boost/optional.hpp>
 
 #include "common/Cond.h"
+#include "common/event_socket.h"
 #include "common/Mutex.h"
 #include "common/Readahead.h"
 #include "common/RWLock.h"
@@ -45,6 +46,7 @@ namespace librbd {
   class LibrbdAdminSocketHook;
   class ImageWatcher;
   class Journal;
+  class AioCompletion;
 
   namespace operation {
   class ResizeRequest;
@@ -101,6 +103,7 @@ namespace librbd {
     RWLock object_map_lock; // protects object map updates and object_map itself
     Mutex async_ops_lock; // protects async_ops and async_requests
     Mutex copyup_list_lock; // protects copyup_waiting_list
+    Mutex completed_reqs_lock; // protects completed_reqs
 
     unsigned extra_read_flags;
 
@@ -140,6 +143,9 @@ namespace librbd {
     xlist<operation::ResizeRequest*> resize_reqs;
 
     AioImageRequestWQ *aio_work_queue;
+    xlist<AioCompletion*> completed_reqs;
+    EventSocket event_socket;
+
     ContextWQ *op_work_queue;
 
     Cond refresh_cond;
diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc
index 9c25ba3add78..0ff32c23ee6d 100644
--- a/src/librbd/internal.cc
+++ b/src/librbd/internal.cc
@@ -12,6 +12,7 @@
 #include "common/ContextCompletion.h"
 #include "common/Throttle.h"
 #include "common/WorkQueue.h"
+#include "common/event_socket.h"
 #include "cls/lock/cls_lock_client.h"
 #include "include/stringify.h"
 
@@ -1991,6 +1992,21 @@ int invoke_async_request(ImageCtx *ictx, const std::string& request_type,
     return ictx->get_flags(ictx->snap_id, flags);
   }
 
+  int set_image_notification(ImageCtx *ictx, int fd, int type)
+  {
+    CephContext *cct = ictx->cct;
+    ldout(cct, 20) << __func__ << " " << ictx << " fd " << fd << " type" << type << dendl;
+
+    int r = ictx_check(ictx);
+    if (r < 0) {
+      return r;
+    }
+
+    if (ictx->event_socket.is_valid())
+      return -EINVAL;
+    return ictx->event_socket.init(fd, type);
+  }
+
   int is_exclusive_lock_owner(ImageCtx *ictx, bool *is_owner)
   {
     RWLock::RLocker l(ictx->owner_lock);
@@ -3416,6 +3432,23 @@ int invoke_async_request(ImageCtx *ictx, const std::string& request_type,
     return r;
   }
 
+  int poll_io_events(ImageCtx *ictx, AioCompletion **comps, int numcomp, int max)
+  {
+    if (numcomp < max || numcomp <= 0)
+      return -EINVAL;
+    CephContext *cct = ictx->cct;
+    ldout(cct, 20) << __func__ << " " << ictx << " numcomp = " << numcomp << " max " << max << dendl;
+    int i = 0;
+    Mutex::Locker l(ictx->completed_reqs_lock);
+    while (i < max) {
+      if (ictx->completed_reqs.empty())
+        break;
+      comps[i++] = ictx->completed_reqs.front();
+      ictx->completed_reqs.pop_front();
+    }
+    return i;
+  }
+
   int metadata_get(ImageCtx *ictx, const string &key, string *value)
   {
     CephContext *cct = ictx->cct;
diff --git a/src/librbd/internal.h b/src/librbd/internal.h
index 04f5a24e87ea..adf03061413b 100644
--- a/src/librbd/internal.h
+++ b/src/librbd/internal.h
@@ -121,6 +121,7 @@ namespace librbd {
   int get_parent_info(ImageCtx *ictx, std::string *parent_pool_name,
 		      std::string *parent_name, std::string *parent_snap_name);
   int get_flags(ImageCtx *ictx, uint64_t *flags);
+  int set_image_notification(ImageCtx *ictx, int fd, int type);
   int is_exclusive_lock_owner(ImageCtx *ictx, bool *is_owner);
 
   int remove(librados::IoCtx& io_ctx, const char *imgname,
@@ -209,6 +210,7 @@ namespace librbd {
 
   int flush(ImageCtx *ictx);
   int invalidate_cache(ImageCtx *ictx);
+  int poll_io_events(ImageCtx *ictx, AioCompletion **comps, int numcomp, int max);
   int metadata_list(ImageCtx *ictx, const string &last, uint64_t max, map<string, bufferlist> *pairs);
   int metadata_get(ImageCtx *ictx, const std::string &key, std::string *value);
   int metadata_set(ImageCtx *ictx, const std::string &key, const std::string &value);
diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc
index e322ad5d167a..20209ede5a38 100644
--- a/src/librbd/librbd.cc
+++ b/src/librbd/librbd.cc
@@ -22,7 +22,6 @@
 #include "common/perf_counters.h"
 #include "common/TracepointProvider.h"
 #include "include/Context.h"
-#include "include/rbd/librbd.hpp"
 #include "osdc/ObjectCacher.h"
 
 #include "librbd/AioCompletion.h"
@@ -284,6 +283,7 @@ namespace librbd {
 							     complete_cb);
     pc = (void *)c;
     c->rbd_comp = this;
+    c->enable_event_notify();
   }
 
   bool RBD::AioCompletion::is_complete()
@@ -502,6 +502,15 @@ namespace librbd {
     return r;
   }
 
+  int Image::set_image_notification(int fd, int type)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    tracepoint(librbd, set_image_notification_enter, ictx, fd, type);
+    int r = librbd::set_image_notification(ictx, fd, type);
+    tracepoint(librbd, set_image_notification_exit, ictx, r);
+    return r;
+  }
+
   int Image::is_exclusive_lock_owner(bool *is_owner)
   {
     ImageCtx *ictx = (ImageCtx *)ctx;
@@ -982,6 +991,20 @@ namespace librbd {
     return r;
   }
 
+  int Image::poll_io_events(RBD::AioCompletion **comps, int numcomp, int max)
+  {
+    AioCompletion *cs[numcomp];
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    tracepoint(librbd, poll_io_events_enter, numcomp, max);
+    int r = librbd::poll_io_events(ictx, cs, numcomp, max);
+    tracepoint(librbd, poll_io_events_exit, r);
+    if (r > 0) {
+      for (int i = 0; i < numcomp; ++i)
+        comps[i] = (RBD::AioCompletion *)cs[i]->rbd_comp;
+    }
+    return r;
+  }
+
   int Image::metadata_get(const std::string &key, std::string *value)
   {
     ImageCtx *ictx = (ImageCtx *)ctx;
@@ -1573,6 +1596,15 @@ extern "C" int rbd_get_flags(rbd_image_t image, uint64_t *flags)
   return r;
 }
 
+extern "C" int rbd_set_image_notification(rbd_image_t image, int fd, int type)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  tracepoint(librbd, set_image_notification_enter, ictx, fd, type);
+  int r = librbd::set_image_notification(ictx, fd, type);
+  tracepoint(librbd, set_image_notification_exit, ictx, r);
+  return r;
+}
+
 extern "C" int rbd_is_exclusive_lock_owner(rbd_image_t image, int *is_owner)
 {
   librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
@@ -2112,6 +2144,20 @@ extern "C" int rbd_invalidate_cache(rbd_image_t image)
   return r;
 }
 
+extern "C" int rbd_poll_io_events(rbd_image_t image, rbd_completion_t *comps, int numcomp, int max)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  librbd::AioCompletion *cs[numcomp];
+  tracepoint(librbd, poll_io_events_enter, numcomp, max);
+  int r = librbd::poll_io_events(ictx, cs, numcomp, max);
+  tracepoint(librbd, poll_io_events_exit, r);
+  if (r > 0) {
+    for (int i = 0; i < r; ++i)
+      comps[i] = cs[i]->rbd_comp;
+  }
+  return r;
+}
+
 extern "C" int rbd_metadata_get(rbd_image_t image, const char *key, char *value, size_t *vallen)
 {
   librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
diff --git a/src/tracing/librbd.tp b/src/tracing/librbd.tp
index f6e1cb1b0bb2..63b64ff12084 100644
--- a/src/tracing/librbd.tp
+++ b/src/tracing/librbd.tp
@@ -504,6 +504,25 @@ TRACEPOINT_EVENT(librbd, invalidate_cache_exit,
     )
 )
 
+TRACEPOINT_EVENT(librbd, poll_io_events_enter,
+    TP_ARGS(
+        void*, imagectx,
+        int, numcomp,
+        int, max),
+    TP_FIELDS(
+        ctf_integer_hex(void*, imagectx, imagectx)
+        ctf_integer(int, numcomp, numcomp)
+        ctf_integer(int, max, max)
+    )
+)
+
+TRACEPOINT_EVENT(librbd, poll_io_events_exit,
+    TP_ARGS(
+        int, retval),
+    TP_FIELDS(
+        ctf_integer(int, retval, retval)
+    )
+)
 TRACEPOINT_EVENT(librbd, metadata_get_enter,
     TP_ARGS(
         void*, imagectx,
@@ -1634,6 +1653,28 @@ TRACEPOINT_EVENT(librbd, get_flags_exit,
     )
 )
 
+TRACEPOINT_EVENT(librbd, set_image_notification_enter,
+    TP_ARGS(
+        void*, imagectx,
+        int, fd,
+        int type),
+    TP_FIELDS(
+      ctf_integer_hex(void*, imagectx, imagectx)
+      ctf_integer(int, fd, fd)
+      ctf_integer(int, type, type)
+    )
+)
+
+TRACEPOINT_EVENT(librbd, set_image_notification_exit,
+    TP_ARGS(
+        void*, imagectx,
+        int, retval)
+    TP_FIELDS(
+      ctf_integer_hex(void*, imagectx, imagectx)
+      ctf_integer(int, retval, retval)
+    )
+)
+
 TRACEPOINT_EVENT(librbd, is_exclusive_lock_owner_enter,
     TP_ARGS(
         void*, imagectx),