Accelio ceph::buffer Extensions

Adds custom buffer::raw type xio_mempool, with hooks into Accelio
memory lifecycle.

The xio_mempool type is non-sharable by default.

Signed-off-by: Matt Benjamin <matt@cohortfs.com>

diff --git a/src/common/buffer.cc b/src/common/buffer.cc
index 2cdfa7714b66dd8e42885827f07ee176c80e985c..18d95df18c7e7ecd5d7d35d5499396e0bcee275b 100644 (file)
--- a/src/common/buffer.cc
+++ b/src/common/buffer.cc
@@ -24,6 +24,9 @@
 #include "common/Mutex.h"
 #include "include/types.h"
 #include "include/compat.h"
+#if defined(HAVE_XIO)
+#include "msg/xio/XioMsg.h"
+#endif
 
 #include <errno.h>
 #include <fstream>
@@ -529,6 +532,59 @@ static simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZER;
     }
   };
 
+#if defined(HAVE_XIO)
+  class buffer::xio_msg_buffer : public buffer::raw {
+  private:
+    XioCompletionHook* m_hook;
+  public:
+    xio_msg_buffer(XioCompletionHook* _m_hook, const char *d,
+       unsigned l) :
+      raw((char*)d, l), m_hook(_m_hook->get()) {}
+
+    bool is_shareable() { return false; }
+    static void operator delete(void *p)
+    {
+      xio_msg_buffer *buf = static_cast<xio_msg_buffer*>(p);
+      // return hook ref (counts against pool);  it appears illegal
+      // to do this in our dtor, because this fires after that
+      buf->m_hook->put();
+    }
+    raw* clone_empty() {
+      return new buffer::raw_char(len);
+    }
+  };
+
+  class buffer::xio_mempool : public buffer::raw {
+  public:
+    struct xio_mempool_obj *mp;
+    xio_mempool(struct xio_mempool_obj *_mp, unsigned l) :
+      raw((char*)mp->addr, l), mp(_mp)
+    { }
+    ~xio_mempool() {}
+    raw* clone_empty() {
+      return new buffer::raw_char(len);
+    }
+  };
+
+  struct xio_mempool_obj* get_xio_mp(const buffer::ptr& bp)
+  {
+    buffer::xio_mempool *mb = dynamic_cast<buffer::xio_mempool*>(bp.get_raw());
+    if (mb) {
+      return mb->mp;
+    }
+    return NULL;
+  }
+
+  buffer::raw* buffer::create_msg(
+      unsigned len, char *buf, XioCompletionHook *m_hook) {
+    XioPool& pool = m_hook->get_pool();
+    buffer::raw* bp =
+      static_cast<buffer::raw*>(pool.alloc(sizeof(xio_msg_buffer)));
+    new (bp) xio_msg_buffer(m_hook, buf, len);
+    return bp;
+  }
+#endif /* HAVE_XIO */
+
   buffer::raw* buffer::copy(const char *c, unsigned len) {
     raw* r = new raw_char(len);
     memcpy(r->data, c, len);

diff --git a/src/include/buffer.h b/src/include/buffer.h
index 6f9adda789d67975891020ede747e109f401da33..5b5eae4b5426b51e31080aa6504638a7fdee5d8b 100644 (file)
--- a/src/include/buffer.h
+++ b/src/include/buffer.h
@@ -58,6 +58,11 @@
   #define CEPH_BUFFER_API
 #endif
 
+#if defined(HAVE_XIO)
+struct xio_mempool_obj;
+class XioCompletionHook;
+#endif
+
 namespace ceph {
 
 class CEPH_BUFFER_API buffer {
@@ -137,6 +142,8 @@ private:
   friend std::ostream& operator<<(std::ostream& out, const raw &r);
 
 public:
+  class xio_mempool;
+  class xio_msg_buffer;
 
   /*
    * named constructors 
@@ -152,6 +159,10 @@ public:
   static raw* create_zero_copy(unsigned len, int fd, int64_t *offset);
   static raw* create_unshareable(unsigned len);
 
+#if defined(HAVE_XIO)
+  static raw* create_msg(unsigned len, char *buf, XioCompletionHook *m_hook);
+#endif
+
   /*
    * a buffer pointer.  references (a subsequence of) a raw buffer.
    */
@@ -513,6 +524,10 @@ public:
   };
 };
 
+#if defined(HAVE_XIO)
+xio_mempool_obj* get_xio_mp(const buffer::ptr& bp);
+#endif
+
 typedef buffer::ptr bufferptr;
 typedef buffer::list bufferlist;
 typedef buffer::hash bufferhash;