ObjectCacher: remove dependency on Objecter

Abstract out how writeback is done with a WritebackHandler object.
For RBD caching, this will be done by librados, but the Client uses
the Objecter directly.

This also requires different locks, since librbd does not have access
to the lock the underlying Objecter uses. Thus, both lock and the
writeback handler are parameters of the ObjectCacher constructor.

Signed-off-by: Josh Durgin <josh.durgin@dreamhost.com>

diff --git a/src/Makefile.am b/src/Makefile.am
index aa3e0a843d5029a29e0e130867201169c4bd00b0..04a0c6f01afaa83429c8a801b0f0d7c6c81787a7 100644 (file)
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -1137,7 +1137,7 @@ libclient_la_SOURCES = \
        client/Inode.cc \
        client/MetaRequest.cc \
        client/Trace.cc
-libclient_la_LIBADD = libcommon.la $(LIBEDIT_LIBS)
+libclient_la_LIBADD = libosdc.la $(LIBEDIT_LIBS)
 noinst_LTLIBRARIES += libclient.la
 
 dist-hook:
@@ -1185,6 +1185,7 @@ noinst_HEADERS = \
         client/fuse_ll.h\
        client/ioctl.h\
         client/hadoop/CephFSInterface.h\
+       client/ObjecterWriteback.h\
        cls_acl.cc\
        cls_crypto.cc\
        common/BackTrace.h\
@@ -1545,6 +1546,7 @@ noinst_HEADERS = \
         osdc/Journaler.h\
         osdc/ObjectCacher.h\
         osdc/Objecter.h\
+       osdc/WritebackHandler.h\
         perfglue/cpu_profiler.h\
         perfglue/heap_profiler.h\
        rgw/rgw_acl.h\

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 73a4fd64739e998c8186cb77b98349f6cdf96c38..7ff97a9dbc8b9d64fbc2ddb60842d724e67e9b37 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -58,8 +58,7 @@ using namespace std;
 #include "mon/MonMap.h"
 
 #include "osdc/Filer.h"
-#include "osdc/Objecter.h"
-#include "osdc/ObjectCacher.h"
+#include "osdc/WritebackHandler.h"
 
 #include "common/Cond.h"
 #include "common/Mutex.h"
@@ -81,6 +80,7 @@ using namespace std;
 #include "Fh.h"
 #include "MetaSession.h"
 #include "MetaRequest.h"
+#include "ObjecterWriteback.h"
 
 #undef dout_prefix
 #define dout_prefix *_dout << "client." << whoami << " "
@@ -150,7 +150,8 @@ Client::Client(Messenger *m, MonClient *mc)
   mdsmap = new MDSMap;
   objecter = new Objecter(cct, messenger, monclient, osdmap, client_lock, timer);
   objecter->set_client_incarnation(0);  // client always 0, for now.
-  objectcacher = new ObjectCacher(cct, objecter, client_lock, 
+  writeback_handler = new ObjecterWriteback(objecter);
+  objectcacher = new ObjectCacher(cct, *writeback_handler, client_lock,
                                  client_flush_set_callback,    // all commit callback
                                  (void*)this);
   filer = new Filer(objecter);
@@ -168,6 +169,11 @@ Client::~Client()
     objectcacher = 0; 
   }
 
+  if (writeback_handler) {
+    delete writeback_handler;
+    writeback_handler = NULL;
+  }
+
   if (filer) { delete filer; filer = 0; }
   if (objecter) { delete objecter; objecter = 0; }
   if (osdmap) { delete osdmap; osdmap = 0; }

diff --git a/src/client/Client.h b/src/client/Client.h
index b7ef4a92065ca026a39f14be120919cb4ba75631..e1febd21fe4ba8f438bdd5f9719b196ade2cccd4 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -65,7 +65,7 @@ class InodeStat;
 
 class Filer;
 class Objecter;
-class ObjectCacher;
+class WritebackHandler;
 
 class PerfCounters;
 
@@ -264,7 +264,8 @@ protected:
   Filer                 *filer;     
   ObjectCacher          *objectcacher;
   Objecter              *objecter;     // (non-blocking) osd interface
-  
+  WritebackHandler      *writeback_handler;
+
   // cache
   hash_map<vinodeno_t, Inode*> inode_map;
   Inode*                 root;

diff --git a/src/client/ObjecterWriteback.h b/src/client/ObjecterWriteback.h
new file mode 100644 (file)
index 0000000..8b5a268
--- /dev/null
+++ b/src/client/ObjecterWriteback.h
@@ -0,0 +1,39 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#ifndef CEPH_OSDC_OBJECTERWRITEBACKHANDLER_H
+#define CEPH_OSDC_OBJECTERWRITEBACKHANDLER_H
+
+#include "osdc/Objecter.h"
+#include "osdc/WritebackHandler.h"
+
+class ObjecterWriteback : public WritebackHandler {
+ public:
+  ObjecterWriteback(Objecter *o) : m_objecter(o) {}
+  virtual ~ObjecterWriteback() {}
+
+  virtual tid_t read(const object_t& oid, const object_locator_t& oloc,
+                    uint64_t off, uint64_t len, snapid_t snapid,
+                    bufferlist *pbl, uint64_t trunc_size,  __u32 trunc_seq,
+                    Context *onfinish) {
+    return m_objecter->read_trunc(oid, oloc, off, len, snapid, pbl, 0,
+                                 trunc_size, trunc_seq, onfinish);
+  }
+
+  virtual tid_t write(const object_t& oid, const object_locator_t& oloc,
+                     uint64_t off, uint64_t len, const SnapContext& snapc,
+                     const bufferlist &bl, utime_t mtime, uint64_t trunc_size,
+                     __u32 trunc_seq, Context *oncommit) {
+    return m_objecter->write_trunc(oid, oloc, off, len, snapc, bl, mtime, 0,
+                                  trunc_size, trunc_seq, NULL, oncommit);
+  }
+
+  virtual tid_t lock(const object_t& oid, const object_locator_t& oloc, int op,
+                    int flags, Context *onack, Context *oncommit) {
+    return m_objecter->lock(oid, oloc, op, flags, onack, oncommit);
+  }
+
+ private:
+  Objecter *m_objecter;
+};
+
+#endif

diff --git a/src/osdc/ObjectCacher.cc b/src/osdc/ObjectCacher.cc
index ffd24cae3c9a2f47be800f7e03181a720d444eed..70d1f3f6a5f2bc22d38a1a99602bd4c925a3893b 100644 (file)
--- a/src/osdc/ObjectCacher.cc
+++ b/src/osdc/ObjectCacher.cc
@@ -3,8 +3,7 @@
 
 #include "msg/Messenger.h"
 #include "ObjectCacher.h"
-#include "Objecter.h"
-
+#include "WritebackHandler.h"
 
 
 /*** ObjectCacher::BufferHead ***/
@@ -14,12 +13,13 @@
 
 #define dout_subsys ceph_subsys_objectcacher
 #undef dout_prefix
-#define dout_prefix *_dout << oc->objecter->messenger->get_myname() << ".objectcacher.object(" << oid << ") "
+#define dout_prefix *_dout << "objectcacher.object(" << oid << ") "
 
 ObjectCacher::
-ObjectCacher(CephContext *cct_, Objecter *o, Mutex& l, flush_set_callback_t flush_callback,
+ObjectCacher(CephContext *cct_, WritebackHandler& wb, Mutex& l,
+            flush_set_callback_t flush_callback,
             void *flush_callback_arg) : 
-    cct(cct_), objecter(o), filer(o), lock(l),
+    cct(cct_), writeback_handler(wb), lock(l),
     flush_set_callback(flush_callback), flush_set_callback_arg(flush_callback_arg),
     flusher_stop(false), flusher_thread(this),
     stat_waiter(0),
@@ -421,7 +421,7 @@ void ObjectCacher::Object::truncate(loff_t s)
 /*** ObjectCacher ***/
 
 #undef dout_prefix
-#define dout_prefix *_dout << objecter->messenger->get_myname() << ".objectcacher "
+#define dout_prefix *_dout << "objectcacher "
 
 /* private */
 
@@ -451,11 +451,10 @@ void ObjectCacher::bh_read(BufferHead *bh)
   ObjectSet *oset = bh->ob->oset;
 
   // go
-  objecter->read_trunc(bh->ob->get_oid(), bh->ob->get_oloc(), 
-                bh->start(), bh->length(), bh->ob->get_snap(),
-                &onfinish->bl, 0,
-                oset->truncate_size, oset->truncate_seq,
-                onfinish);
+  writeback_handler.read(bh->ob->get_oid(), bh->ob->get_oloc(),
+                        bh->start(), bh->length(), bh->ob->get_snap(),
+                        &onfinish->bl, oset->truncate_size, oset->truncate_seq,
+                        onfinish);
 }
 
 void ObjectCacher::bh_read_finish(int64_t poolid, sobject_t oid, loff_t start,
@@ -551,11 +550,11 @@ void ObjectCacher::bh_write(BufferHead *bh)
   ObjectSet *oset = bh->ob->oset;
 
   // go
-  tid_t tid = objecter->write_trunc(bh->ob->get_oid(), bh->ob->get_oloc(),
-                             bh->start(), bh->length(),
-                             bh->snapc, bh->bl, bh->last_write, 0,
-                             oset->truncate_size, oset->truncate_seq,
-                                   NULL, oncommit);
+  tid_t tid = writeback_handler.write(bh->ob->get_oid(), bh->ob->get_oloc(),
+                                     bh->start(), bh->length(),
+                                     bh->snapc, bh->bl, bh->last_write,
+                                     oset->truncate_size, oset->truncate_seq,
+                                     oncommit);
 
   // set bh last_write_tid
   oncommit->tid = tid;
@@ -1080,7 +1079,9 @@ void ObjectCacher::rdlock(Object *o)
     
     commit->tid = 
       ack->tid = 
-      o->last_write_tid = objecter->lock(o->get_oid(), o->get_oloc(), CEPH_OSD_OP_RDLOCK, 0, ack, commit);
+      o->last_write_tid = writeback_handler.lock(o->get_oid(), o->get_oloc(),
+                                                CEPH_OSD_OP_RDLOCK, 0,
+                                                ack, commit);
   }
   
   // stake our claim.
@@ -1125,7 +1126,8 @@ void ObjectCacher::wrlock(Object *o)
     
     commit->tid = 
       ack->tid = 
-      o->last_write_tid = objecter->lock(o->get_oid(), o->get_oloc(), op, 0, ack, commit);
+      o->last_write_tid = writeback_handler.lock(o->get_oid(), o->get_oloc(),
+                                                op, 0, ack, commit);
   }
   
   // stake our claim.
@@ -1170,7 +1172,9 @@ void ObjectCacher::rdunlock(Object *o)
                                             o->get_soid(), 0, 0);
   commit->tid = 
     lockack->tid = 
-    o->last_write_tid = objecter->lock(o->get_oid(), o->get_oloc(), CEPH_OSD_OP_RDUNLOCK, 0, lockack, commit);
+    o->last_write_tid = writeback_handler.lock(o->get_oid(), o->get_oloc(),
+                                              CEPH_OSD_OP_RDUNLOCK, 0,
+                                              lockack, commit);
 }
 
 void ObjectCacher::wrunlock(Object *o)
@@ -1203,7 +1207,8 @@ void ObjectCacher::wrunlock(Object *o)
                                             o->get_soid(), 0, 0);
   commit->tid = 
     lockack->tid = 
-    o->last_write_tid = objecter->lock(o->get_oid(), o->get_oloc(), op, 0, lockack, commit);
+    o->last_write_tid = writeback_handler.lock(o->get_oid(), o->get_oloc(),
+                                              op, 0, lockack, commit);
 }
 
 

diff --git a/src/osdc/ObjectCacher.h b/src/osdc/ObjectCacher.h
index 89c495a1f1da02f54dbc3085556e2d836470e9e3..184a49bb23407d5003b1dab6ced0f6d5097451a0 100644 (file)
--- a/src/osdc/ObjectCacher.h
+++ b/src/osdc/ObjectCacher.h
@@ -15,7 +15,7 @@
 #include "Filer.h"
 
 class CephContext;
-class Objecter;
+class WritebackHandler;
 
 class ObjectCacher {
  public:
@@ -267,11 +267,9 @@ class ObjectCacher {
 
   // ******* ObjectCacher *********
   // ObjectCacher fields
- public:
-  Objecter *objecter;
-  Filer filer;
-
  private:
+  WritebackHandler& writeback_handler;
+
   Mutex& lock;
   
   flush_set_callback_t flush_set_callback;
@@ -517,7 +515,7 @@ class ObjectCacher {
 
 
  public:
-  ObjectCacher(CephContext *cct_, Objecter *o, Mutex& l,
+  ObjectCacher(CephContext *cct_, WritebackHandler& wb, Mutex& l,
               flush_set_callback_t flush_callback,
               void *flush_callback_arg);
   ~ObjectCacher() {

diff --git a/src/osdc/WritebackHandler.h b/src/osdc/WritebackHandler.h
new file mode 100644 (file)
index 0000000..8a00927
--- /dev/null
+++ b/src/osdc/WritebackHandler.h
@@ -0,0 +1,29 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#ifndef CEPH_OSDC_WRITEBACKHANDLER_H
+#define CEPH_OSDC_WRITEBACKHANDLER_H
+
+#include "include/Context.h"
+#include "include/types.h"
+#include "osd/osd_types.h"
+
+class WritebackHandler {
+ public:
+  WritebackHandler() {}
+  virtual ~WritebackHandler() {}
+
+  virtual tid_t read(const object_t& oid, const object_locator_t& oloc,
+                    uint64_t off, uint64_t len, snapid_t snapid,
+                    bufferlist *pbl, uint64_t trunc_size,  __u32 trunc_seq,
+                    Context *onfinish) = 0;
+  virtual tid_t write(const object_t& oid, const object_locator_t& oloc,
+                     uint64_t off, uint64_t len, const SnapContext& snapc,
+                     const bufferlist &bl, utime_t mtime, uint64_t trunc_size,
+                     __u32 trunc_seq, Context *oncommit) = 0;
+  virtual tid_t lock(const object_t& oid, const object_locator_t& oloc, int op,
+                    int flags, Context *onack, Context *oncommit) {
+    assert(0 == "this WritebackHandler does not support the lock operation");
+  }
+};
+
+#endif