From d6dbff01a3c652d2710e2b647e454683d56b5960 Mon Sep 17 00:00:00 2001 From: Josh Durgin Date: Mon, 9 Apr 2012 13:04:41 -0700 Subject: [PATCH] ObjectCacher: remove dependency on Objecter Abstract out how writeback is done with a WritebackHandler object. For RBD caching, this will be done by librados, but the Client uses the Objecter directly. This also requires different locks, since librbd does not have access to the lock the underlying Objecter uses. Thus, both lock and the writeback handler are parameters of the ObjectCacher constructor. Signed-off-by: Josh Durgin --- src/Makefile.am | 4 ++- src/client/Client.cc | 12 ++++++--- src/client/Client.h | 5 ++-- src/client/ObjecterWriteback.h | 39 +++++++++++++++++++++++++++++ src/osdc/ObjectCacher.cc | 45 +++++++++++++++++++--------------- src/osdc/ObjectCacher.h | 10 +++----- src/osdc/WritebackHandler.h | 29 ++++++++++++++++++++++ 7 files changed, 112 insertions(+), 32 deletions(-) create mode 100644 src/client/ObjecterWriteback.h create mode 100644 src/osdc/WritebackHandler.h diff --git a/src/Makefile.am b/src/Makefile.am index aa3e0a843d502..04a0c6f01afaa 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1137,7 +1137,7 @@ libclient_la_SOURCES = \ client/Inode.cc \ client/MetaRequest.cc \ client/Trace.cc -libclient_la_LIBADD = libcommon.la $(LIBEDIT_LIBS) +libclient_la_LIBADD = libosdc.la $(LIBEDIT_LIBS) noinst_LTLIBRARIES += libclient.la dist-hook: @@ -1185,6 +1185,7 @@ noinst_HEADERS = \ client/fuse_ll.h\ client/ioctl.h\ client/hadoop/CephFSInterface.h\ + client/ObjecterWriteback.h\ cls_acl.cc\ cls_crypto.cc\ common/BackTrace.h\ @@ -1545,6 +1546,7 @@ noinst_HEADERS = \ osdc/Journaler.h\ osdc/ObjectCacher.h\ osdc/Objecter.h\ + osdc/WritebackHandler.h\ perfglue/cpu_profiler.h\ perfglue/heap_profiler.h\ rgw/rgw_acl.h\ diff --git a/src/client/Client.cc b/src/client/Client.cc index 73a4fd64739e9..7ff97a9dbc8b9 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -58,8 +58,7 @@ using namespace std; #include "mon/MonMap.h" #include "osdc/Filer.h" -#include "osdc/Objecter.h" -#include "osdc/ObjectCacher.h" +#include "osdc/WritebackHandler.h" #include "common/Cond.h" #include "common/Mutex.h" @@ -81,6 +80,7 @@ using namespace std; #include "Fh.h" #include "MetaSession.h" #include "MetaRequest.h" +#include "ObjecterWriteback.h" #undef dout_prefix #define dout_prefix *_dout << "client." << whoami << " " @@ -150,7 +150,8 @@ Client::Client(Messenger *m, MonClient *mc) mdsmap = new MDSMap; objecter = new Objecter(cct, messenger, monclient, osdmap, client_lock, timer); objecter->set_client_incarnation(0); // client always 0, for now. - objectcacher = new ObjectCacher(cct, objecter, client_lock, + writeback_handler = new ObjecterWriteback(objecter); + objectcacher = new ObjectCacher(cct, *writeback_handler, client_lock, client_flush_set_callback, // all commit callback (void*)this); filer = new Filer(objecter); @@ -168,6 +169,11 @@ Client::~Client() objectcacher = 0; } + if (writeback_handler) { + delete writeback_handler; + writeback_handler = NULL; + } + if (filer) { delete filer; filer = 0; } if (objecter) { delete objecter; objecter = 0; } if (osdmap) { delete osdmap; osdmap = 0; } diff --git a/src/client/Client.h b/src/client/Client.h index b7ef4a92065ca..e1febd21fe4ba 100644 --- a/src/client/Client.h +++ b/src/client/Client.h @@ -65,7 +65,7 @@ class InodeStat; class Filer; class Objecter; -class ObjectCacher; +class WritebackHandler; class PerfCounters; @@ -264,7 +264,8 @@ protected: Filer *filer; ObjectCacher *objectcacher; Objecter *objecter; // (non-blocking) osd interface - + WritebackHandler *writeback_handler; + // cache hash_map inode_map; Inode* root; diff --git a/src/client/ObjecterWriteback.h b/src/client/ObjecterWriteback.h new file mode 100644 index 0000000000000..8b5a2684da804 --- /dev/null +++ b/src/client/ObjecterWriteback.h @@ -0,0 +1,39 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +#ifndef CEPH_OSDC_OBJECTERWRITEBACKHANDLER_H +#define CEPH_OSDC_OBJECTERWRITEBACKHANDLER_H + +#include "osdc/Objecter.h" +#include "osdc/WritebackHandler.h" + +class ObjecterWriteback : public WritebackHandler { + public: + ObjecterWriteback(Objecter *o) : m_objecter(o) {} + virtual ~ObjecterWriteback() {} + + virtual tid_t read(const object_t& oid, const object_locator_t& oloc, + uint64_t off, uint64_t len, snapid_t snapid, + bufferlist *pbl, uint64_t trunc_size, __u32 trunc_seq, + Context *onfinish) { + return m_objecter->read_trunc(oid, oloc, off, len, snapid, pbl, 0, + trunc_size, trunc_seq, onfinish); + } + + virtual tid_t write(const object_t& oid, const object_locator_t& oloc, + uint64_t off, uint64_t len, const SnapContext& snapc, + const bufferlist &bl, utime_t mtime, uint64_t trunc_size, + __u32 trunc_seq, Context *oncommit) { + return m_objecter->write_trunc(oid, oloc, off, len, snapc, bl, mtime, 0, + trunc_size, trunc_seq, NULL, oncommit); + } + + virtual tid_t lock(const object_t& oid, const object_locator_t& oloc, int op, + int flags, Context *onack, Context *oncommit) { + return m_objecter->lock(oid, oloc, op, flags, onack, oncommit); + } + + private: + Objecter *m_objecter; +}; + +#endif diff --git a/src/osdc/ObjectCacher.cc b/src/osdc/ObjectCacher.cc index ffd24cae3c9a2..70d1f3f6a5f2b 100644 --- a/src/osdc/ObjectCacher.cc +++ b/src/osdc/ObjectCacher.cc @@ -3,8 +3,7 @@ #include "msg/Messenger.h" #include "ObjectCacher.h" -#include "Objecter.h" - +#include "WritebackHandler.h" /*** ObjectCacher::BufferHead ***/ @@ -14,12 +13,13 @@ #define dout_subsys ceph_subsys_objectcacher #undef dout_prefix -#define dout_prefix *_dout << oc->objecter->messenger->get_myname() << ".objectcacher.object(" << oid << ") " +#define dout_prefix *_dout << "objectcacher.object(" << oid << ") " ObjectCacher:: -ObjectCacher(CephContext *cct_, Objecter *o, Mutex& l, flush_set_callback_t flush_callback, +ObjectCacher(CephContext *cct_, WritebackHandler& wb, Mutex& l, + flush_set_callback_t flush_callback, void *flush_callback_arg) : - cct(cct_), objecter(o), filer(o), lock(l), + cct(cct_), writeback_handler(wb), lock(l), flush_set_callback(flush_callback), flush_set_callback_arg(flush_callback_arg), flusher_stop(false), flusher_thread(this), stat_waiter(0), @@ -421,7 +421,7 @@ void ObjectCacher::Object::truncate(loff_t s) /*** ObjectCacher ***/ #undef dout_prefix -#define dout_prefix *_dout << objecter->messenger->get_myname() << ".objectcacher " +#define dout_prefix *_dout << "objectcacher " /* private */ @@ -451,11 +451,10 @@ void ObjectCacher::bh_read(BufferHead *bh) ObjectSet *oset = bh->ob->oset; // go - objecter->read_trunc(bh->ob->get_oid(), bh->ob->get_oloc(), - bh->start(), bh->length(), bh->ob->get_snap(), - &onfinish->bl, 0, - oset->truncate_size, oset->truncate_seq, - onfinish); + writeback_handler.read(bh->ob->get_oid(), bh->ob->get_oloc(), + bh->start(), bh->length(), bh->ob->get_snap(), + &onfinish->bl, oset->truncate_size, oset->truncate_seq, + onfinish); } void ObjectCacher::bh_read_finish(int64_t poolid, sobject_t oid, loff_t start, @@ -551,11 +550,11 @@ void ObjectCacher::bh_write(BufferHead *bh) ObjectSet *oset = bh->ob->oset; // go - tid_t tid = objecter->write_trunc(bh->ob->get_oid(), bh->ob->get_oloc(), - bh->start(), bh->length(), - bh->snapc, bh->bl, bh->last_write, 0, - oset->truncate_size, oset->truncate_seq, - NULL, oncommit); + tid_t tid = writeback_handler.write(bh->ob->get_oid(), bh->ob->get_oloc(), + bh->start(), bh->length(), + bh->snapc, bh->bl, bh->last_write, + oset->truncate_size, oset->truncate_seq, + oncommit); // set bh last_write_tid oncommit->tid = tid; @@ -1080,7 +1079,9 @@ void ObjectCacher::rdlock(Object *o) commit->tid = ack->tid = - o->last_write_tid = objecter->lock(o->get_oid(), o->get_oloc(), CEPH_OSD_OP_RDLOCK, 0, ack, commit); + o->last_write_tid = writeback_handler.lock(o->get_oid(), o->get_oloc(), + CEPH_OSD_OP_RDLOCK, 0, + ack, commit); } // stake our claim. @@ -1125,7 +1126,8 @@ void ObjectCacher::wrlock(Object *o) commit->tid = ack->tid = - o->last_write_tid = objecter->lock(o->get_oid(), o->get_oloc(), op, 0, ack, commit); + o->last_write_tid = writeback_handler.lock(o->get_oid(), o->get_oloc(), + op, 0, ack, commit); } // stake our claim. @@ -1170,7 +1172,9 @@ void ObjectCacher::rdunlock(Object *o) o->get_soid(), 0, 0); commit->tid = lockack->tid = - o->last_write_tid = objecter->lock(o->get_oid(), o->get_oloc(), CEPH_OSD_OP_RDUNLOCK, 0, lockack, commit); + o->last_write_tid = writeback_handler.lock(o->get_oid(), o->get_oloc(), + CEPH_OSD_OP_RDUNLOCK, 0, + lockack, commit); } void ObjectCacher::wrunlock(Object *o) @@ -1203,7 +1207,8 @@ void ObjectCacher::wrunlock(Object *o) o->get_soid(), 0, 0); commit->tid = lockack->tid = - o->last_write_tid = objecter->lock(o->get_oid(), o->get_oloc(), op, 0, lockack, commit); + o->last_write_tid = writeback_handler.lock(o->get_oid(), o->get_oloc(), + op, 0, lockack, commit); } diff --git a/src/osdc/ObjectCacher.h b/src/osdc/ObjectCacher.h index 89c495a1f1da0..184a49bb23407 100644 --- a/src/osdc/ObjectCacher.h +++ b/src/osdc/ObjectCacher.h @@ -15,7 +15,7 @@ #include "Filer.h" class CephContext; -class Objecter; +class WritebackHandler; class ObjectCacher { public: @@ -267,11 +267,9 @@ class ObjectCacher { // ******* ObjectCacher ********* // ObjectCacher fields - public: - Objecter *objecter; - Filer filer; - private: + WritebackHandler& writeback_handler; + Mutex& lock; flush_set_callback_t flush_set_callback; @@ -517,7 +515,7 @@ class ObjectCacher { public: - ObjectCacher(CephContext *cct_, Objecter *o, Mutex& l, + ObjectCacher(CephContext *cct_, WritebackHandler& wb, Mutex& l, flush_set_callback_t flush_callback, void *flush_callback_arg); ~ObjectCacher() { diff --git a/src/osdc/WritebackHandler.h b/src/osdc/WritebackHandler.h new file mode 100644 index 0000000000000..8a00927063baa --- /dev/null +++ b/src/osdc/WritebackHandler.h @@ -0,0 +1,29 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +#ifndef CEPH_OSDC_WRITEBACKHANDLER_H +#define CEPH_OSDC_WRITEBACKHANDLER_H + +#include "include/Context.h" +#include "include/types.h" +#include "osd/osd_types.h" + +class WritebackHandler { + public: + WritebackHandler() {} + virtual ~WritebackHandler() {} + + virtual tid_t read(const object_t& oid, const object_locator_t& oloc, + uint64_t off, uint64_t len, snapid_t snapid, + bufferlist *pbl, uint64_t trunc_size, __u32 trunc_seq, + Context *onfinish) = 0; + virtual tid_t write(const object_t& oid, const object_locator_t& oloc, + uint64_t off, uint64_t len, const SnapContext& snapc, + const bufferlist &bl, utime_t mtime, uint64_t trunc_size, + __u32 trunc_seq, Context *oncommit) = 0; + virtual tid_t lock(const object_t& oid, const object_locator_t& oloc, int op, + int flags, Context *onack, Context *oncommit) { + assert(0 == "this WritebackHandler does not support the lock operation"); + } +}; + +#endif -- 2.39.5