From: John Spray Date: Wed, 12 Feb 2014 12:19:57 +0000 (+0000) Subject: mds: Create MDSUtility as base for Dumper/Resetter X-Git-Tag: v0.79~189^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=7a985df6339499052bb7bad2df214477e7e1580a;p=ceph.git mds: Create MDSUtility as base for Dumper/Resetter These classes has lots of similar functionality and we may want to add more similar things in the future. Also, get a copy of the MDS map at startup to learn the ID of the metadata pool instead of assuming it's the default. Signed-off-by: John Spray --- diff --git a/src/ceph_mds.cc b/src/ceph_mds.cc index 23be20827f71..14b84097bc27 100644 --- a/src/ceph_mds.cc +++ b/src/ceph_mds.cc @@ -71,46 +71,27 @@ void usage() static int do_cmds_special_action(const std::string &action, const std::string &dump_file, int rank) { - common_init_finish(g_ceph_context); - Messenger *messenger = Messenger::create(g_ceph_context, - entity_name_t::CLIENT(), "mds", - getpid()); - int r = messenger->bind(g_conf->public_addr); - if (r < 0) - return r; - MonClient mc(g_ceph_context); - if (mc.build_initial_monmap() < 0) - return -1; + common_init_finish(g_ceph_context, CINIT_FLAG_NO_DAEMON_ACTIONS); if (action == "dump-journal") { dout(0) << "dumping journal for mds." << rank << " to " << dump_file << dendl; - Dumper *journal_dumper = new Dumper(messenger, &mc); - journal_dumper->init(rank); - journal_dumper->dump(dump_file.c_str()); - mc.shutdown(); - messenger->shutdown(); - messenger->wait(); - } - else if (action == "undump-journal") { + Dumper journal_dumper; + journal_dumper.init(rank); + journal_dumper.dump(dump_file.c_str()); + journal_dumper.shutdown(); + } else if (action == "undump-journal") { dout(0) << "undumping journal for mds." << rank << " from " << dump_file << dendl; - Dumper *journal_dumper = new Dumper(messenger, &mc); - journal_dumper->init(rank); - journal_dumper->undump(dump_file.c_str()); - mc.shutdown(); - messenger->shutdown(); - messenger->wait(); - } - else if (action == "reset-journal") { + Dumper journal_dumper; + journal_dumper.init(rank); + journal_dumper.undump(dump_file.c_str()); + journal_dumper.shutdown(); + } else if (action == "reset-journal") { dout(0) << "resetting journal" << dendl; - Resetter *jr = new Resetter(messenger, &mc); - jr->init(rank); - jr->reset(); - mc.shutdown(); - messenger->shutdown(); - messenger->wait(); - } - - else { + Resetter resetter; + resetter.init(rank); + resetter.reset(); + resetter.shutdown(); + } else { assert(0); } return 0; diff --git a/src/mds/Dumper.cc b/src/mds/Dumper.cc index bec44c4ba1dc..fd6aef9579aa 100644 --- a/src/mds/Dumper.cc +++ b/src/mds/Dumper.cc @@ -22,79 +22,35 @@ #include "common/safe_io.h" #include "mds/Dumper.h" #include "mds/mdstypes.h" -#include "mon/MonClient.h" #include "osdc/Journaler.h" #define dout_subsys ceph_subsys_mds -Dumper::~Dumper() -{ -} -bool Dumper::ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, - bool force_new) +int Dumper::init(int rank_) { - if (dest_type == CEPH_ENTITY_TYPE_MON) - return true; + rank = rank_; - if (force_new) { - if (monc->wait_auth_rotating(10) < 0) - return false; + int r = MDSUtility::init(); + if (r < 0) { + return r; } - *authorizer = monc->auth->build_authorizer(dest_type); - return *authorizer != NULL; -} - -void Dumper::init(int rank) -{ inodeno_t ino = MDS_INO_LOG_OFFSET + rank; - unsigned pg_pool = MDS_METADATA_POOL; - osdmap = new OSDMap(); - objecter = new Objecter(g_ceph_context, messenger, monc, osdmap, lock, timer, - 0, 0); - journaler = new Journaler(ino, pg_pool, CEPH_FS_ONDISK_MAGIC, + journaler = new Journaler(ino, mdsmap->get_metadata_pool(), CEPH_FS_ONDISK_MAGIC, objecter, 0, 0, &timer); - - objecter->set_client_incarnation(0); - - messenger->add_dispatcher_head(this); - messenger->start(); - - monc->set_want_keys(CEPH_ENTITY_TYPE_MON|CEPH_ENTITY_TYPE_OSD|CEPH_ENTITY_TYPE_MDS); - monc->set_messenger(messenger); - monc->init(); - monc->authenticate(); - - client_t whoami = monc->get_global_id(); - messenger->set_myname(entity_name_t::CLIENT(whoami.v)); - - objecter->init_unlocked(); - lock.Lock(); - objecter->init_locked(); - lock.Unlock(); - objecter->wait_for_osd_map(); - timer.init(); + return 0; } -void Dumper::shutdown() -{ - lock.Lock(); - timer.shutdown(); - objecter->shutdown_locked(); - lock.Unlock(); - objecter->shutdown_unlocked(); -} void Dumper::dump(const char *dump_file) { bool done = false; - Cond cond; int r = 0; - int rank = strtol(g_conf->name.get_id().c_str(), 0, 0); - inodeno_t ino = MDS_INO_LOG_OFFSET + rank; + Cond cond; Mutex localLock("dump:lock"); + lock.Lock(); journaler->recover(new C_SafeCond(&localLock, &cond, &done, &r)); lock.Unlock(); @@ -105,7 +61,6 @@ void Dumper::dump(const char *dump_file) if (r < 0) { // Error derr << "error on recovery: " << cpp_strerror(r) << dendl; - shutdown(); } else { dout(10) << "completed journal recovery" << dendl; } @@ -113,10 +68,13 @@ void Dumper::dump(const char *dump_file) uint64_t start = journaler->get_read_pos(); uint64_t end = journaler->get_write_pos(); uint64_t len = end-start; + inodeno_t ino = MDS_INO_LOG_OFFSET + rank; + cout << "journal is " << start << "~" << len << std::endl; Filer filer(objecter); bufferlist bl; + lock.Lock(); filer.read(ino, &journaler->get_layout(), CEPH_NOSNAP, start, len, &bl, 0, new C_SafeCond(&localLock, &cond, &done)); @@ -155,8 +113,6 @@ void Dumper::dump(const char *dump_file) int err = errno; derr << "unable to open " << dump_file << ": " << cpp_strerror(err) << dendl; } - - shutdown(); } void Dumper::undump(const char *dump_file) @@ -187,7 +143,6 @@ void Dumper::undump(const char *dump_file) cout << "start " << start << " len " << len << std::endl; inodeno_t ino = MDS_INO_LOG_OFFSET + rank; - unsigned pg_pool = MDS_METADATA_POOL; Journaler::Header h; h.trimmed_pos = start; @@ -196,13 +151,13 @@ void Dumper::undump(const char *dump_file) h.magic = CEPH_FS_ONDISK_MAGIC; h.layout = g_default_file_layout; - h.layout.fl_pg_pool = pg_pool; + h.layout.fl_pg_pool = mdsmap->get_metadata_pool(); bufferlist hbl; ::encode(h, hbl); object_t oid = file_object_t(ino, 0); - object_locator_t oloc(pg_pool); + object_locator_t oloc(mdsmap->get_metadata_pool()); SnapContext snapc; bool done = false; @@ -242,5 +197,3 @@ void Dumper::undump(const char *dump_file) VOID_TEMP_FAILURE_RETRY(::close(fd)); cout << "done." << std::endl; } - - diff --git a/src/mds/Dumper.h b/src/mds/Dumper.h index 794ae0c8aa73..13e3adddd8e9 100644 --- a/src/mds/Dumper.h +++ b/src/mds/Dumper.h @@ -14,12 +14,9 @@ #ifndef JOURNAL_DUMPER_H_ #define JOURNAL_DUMPER_H_ -#include "osd/OSDMap.h" -#include "osdc/Objecter.h" + +#include "mds/MDSUtility.h" #include "osdc/Journaler.h" -#include "msg/Dispatcher.h" -#include "msg/Messenger.h" -#include "auth/Auth.h" /** * This class lets you dump out an mds journal for troubleshooting or whatever. @@ -29,58 +26,18 @@ * of the file to dump to. */ -class Dumper : public Dispatcher { -public: - Objecter *objecter; +class Dumper : public MDSUtility { +private: Journaler *journaler; - OSDMap *osdmap; - Messenger *messenger; - MonClient *monc; - Mutex lock; - SafeTimer timer; - int rank; - /* - * The messenger should be a valid Messenger. You should call bind() - * before passing it in, but not do anything else. - * The MonClient needs to be valid, and you should have called - * build_initial_monmap(). - */ - Dumper(Messenger *messenger_, MonClient *monc_) : - Dispatcher(messenger_->cct), - objecter(NULL), - journaler(NULL), - osdmap(NULL), - messenger(messenger_), - monc(monc_), - lock("Dumper::lock"), - timer(g_ceph_context, lock), - rank(-1) - { } +public: + Dumper() : journaler(NULL), rank(-1) + {} - virtual ~Dumper(); + void handle_mds_map(MMDSMap* m); - bool ms_dispatch(Message *m) { - Mutex::Locker locker(lock); - switch (m->get_type()) { - case CEPH_MSG_OSD_OPREPLY: - objecter->handle_osd_op_reply((MOSDOpReply *)m); - break; - case CEPH_MSG_OSD_MAP: - objecter->handle_osd_map((MOSDMap*)m); - break; - default: - return false; - } - return true; - } - bool ms_handle_reset(Connection *con) { return false; } - void ms_handle_remote_reset(Connection *con) {} - bool ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, - bool force_new); - void init(int rank); - void shutdown(); + int init(int rank); void dump(const char *dumpfile); void undump(const char *dumpfile); }; diff --git a/src/mds/LogEvent.h b/src/mds/LogEvent.h index fdf145c85eaf..e7e8a62e06ab 100644 --- a/src/mds/LogEvent.h +++ b/src/mds/LogEvent.h @@ -83,6 +83,7 @@ protected: virtual void encode(bufferlist& bl) const = 0; virtual void decode(bufferlist::iterator &bl) = 0; static LogEvent *decode(bufferlist &bl); + virtual void dump(Formatter *f) const = 0; void encode_with_header(bufferlist& bl) { ::encode(EVENT_NEW_ENCODING, bl); diff --git a/src/mds/MDSUtility.cc b/src/mds/MDSUtility.cc new file mode 100644 index 000000000000..09be280f3603 --- /dev/null +++ b/src/mds/MDSUtility.cc @@ -0,0 +1,160 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2014 John Spray + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#include "mds/MDSUtility.h" +#include "mon/MonClient.h" + +#define dout_subsys ceph_subsys_mds + + +MDSUtility::MDSUtility() : + Dispatcher(g_ceph_context), + objecter(NULL), + lock("MDSUtility::lock"), + timer(g_ceph_context, lock), + waiting_for_mds_map(NULL) +{ + monc = new MonClient(g_ceph_context); + messenger = Messenger::create(g_ceph_context, entity_name_t::CLIENT(), "mds", getpid()); + mdsmap = new MDSMap(); + osdmap = new OSDMap(); + objecter = new Objecter(g_ceph_context, messenger, monc, osdmap, lock, timer, 0, 0); +} + + +MDSUtility::~MDSUtility() +{ + delete objecter; + delete monc; + delete messenger; + delete osdmap; + delete mdsmap; + assert(waiting_for_mds_map == NULL); +} + + +int MDSUtility::init() +{ + // Initialize Messenger + int r = messenger->bind(g_conf->public_addr); + if (r < 0) + return r; + + messenger->add_dispatcher_head(this); + messenger->start(); + + // Initialize MonClient + if (monc->build_initial_monmap() < 0) + return -1; + + monc->set_want_keys(CEPH_ENTITY_TYPE_MON|CEPH_ENTITY_TYPE_OSD|CEPH_ENTITY_TYPE_MDS); + monc->set_messenger(messenger); + monc->init(); + r = monc->authenticate(); + if (r < 0) { + derr << "Authentication failed, did you specify an MDS ID with a valid keyring?" << dendl; + return r; + } + + client_t whoami = monc->get_global_id(); + messenger->set_myname(entity_name_t::CLIENT(whoami.v)); + + // Initialize Objecter and wait for OSD map + objecter->set_client_incarnation(0); + objecter->init_unlocked(); + lock.Lock(); + objecter->init_locked(); + lock.Unlock(); + objecter->wait_for_osd_map(); + timer.init(); + + // Prepare to receive MDS map and request it + Mutex init_lock("MDSUtility:init"); + Cond cond; + bool done = false; + assert(!mdsmap->get_epoch()); + lock.Lock(); + waiting_for_mds_map = new C_SafeCond(&init_lock, &cond, &done, NULL); + lock.Unlock(); + monc->sub_want("mdsmap", 0, CEPH_SUBSCRIBE_ONETIME); + monc->renew_subs(); + + // Wait for MDS map + dout(4) << "waiting for MDS map..." << dendl; + init_lock.Lock(); + while (!done) + cond.Wait(init_lock); + init_lock.Unlock(); + dout(4) << "Got MDS map " << mdsmap->get_epoch() << dendl; + + return 0; +} + + +void MDSUtility::shutdown() +{ + lock.Lock(); + timer.shutdown(); + objecter->shutdown_locked(); + lock.Unlock(); + objecter->shutdown_unlocked(); + monc->shutdown(); + messenger->shutdown(); + messenger->wait(); +} + + +bool MDSUtility::ms_dispatch(Message *m) +{ + Mutex::Locker locker(lock); + switch (m->get_type()) { + case CEPH_MSG_OSD_OPREPLY: + objecter->handle_osd_op_reply((MOSDOpReply *)m); + break; + case CEPH_MSG_OSD_MAP: + objecter->handle_osd_map((MOSDMap*)m); + break; + case CEPH_MSG_MDS_MAP: + handle_mds_map((MMDSMap*)m); + break; + default: + return false; + } + return true; +} + + +void MDSUtility::handle_mds_map(MMDSMap* m) +{ + mdsmap->decode(m->get_encoded()); + if (waiting_for_mds_map) { + waiting_for_mds_map->complete(0); + waiting_for_mds_map = NULL; + } +} + + +bool MDSUtility::ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, + bool force_new) +{ + if (dest_type == CEPH_ENTITY_TYPE_MON) + return true; + + if (force_new) { + if (monc->wait_auth_rotating(10) < 0) + return false; + } + + *authorizer = monc->auth->build_authorizer(dest_type); + return *authorizer != NULL; +} diff --git a/src/mds/MDSUtility.h b/src/mds/MDSUtility.h new file mode 100644 index 000000000000..d3f938e81713 --- /dev/null +++ b/src/mds/MDSUtility.h @@ -0,0 +1,58 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2014 John Spray + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#ifndef MDS_UTILITY_H_ +#define MDS_UTILITY_H_ + +#include "osd/OSDMap.h" +#include "osdc/Objecter.h" +#include "mds/MDSMap.h" +#include "messages/MMDSMap.h" +#include "msg/Dispatcher.h" +#include "msg/Messenger.h" +#include "auth/Auth.h" + +/// MDS Utility +/** + * This class is the parent for MDS utilities, i.e. classes that + * need access the objects belonging to the MDS without actually + * acting as an MDS daemon themselves. + */ +class MDSUtility : public Dispatcher { +protected: + Objecter *objecter; + OSDMap *osdmap; + MDSMap *mdsmap; + Messenger *messenger; + MonClient *monc; + + Mutex lock; + SafeTimer timer; + + Context *waiting_for_mds_map; + +public: + MDSUtility(); + ~MDSUtility(); + + void handle_mds_map(MMDSMap* m); + bool ms_dispatch(Message *m); + bool ms_handle_reset(Connection *con) { return false; } + void ms_handle_remote_reset(Connection *con) {} + bool ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, + bool force_new); + int init(); + void shutdown(); +}; + +#endif /* MDS_UTILITY_H_ */ diff --git a/src/mds/Makefile.am b/src/mds/Makefile.am index 05d0e1a24296..4ee35003ace0 100644 --- a/src/mds/Makefile.am +++ b/src/mds/Makefile.am @@ -27,7 +27,8 @@ libmds_la_SOURCES = \ mds/SnapServer.cc \ mds/snap.cc \ mds/SessionMap.cc \ - mds/MDLog.cc + mds/MDLog.cc \ + mds/MDSUtility.cc libmds_la_LIBADD = $(LIBOSDC) noinst_LTLIBRARIES += libmds.la @@ -57,6 +58,7 @@ noinst_HEADERS += \ mds/MDSTable.h \ mds/MDSTableServer.h \ mds/MDSTableClient.h \ + mds/MDSUtility.h \ mds/Mutation.h \ mds/Migrator.h \ mds/Resetter.h \ diff --git a/src/mds/Resetter.cc b/src/mds/Resetter.cc index 74da069e3437..22839eccf732 100644 --- a/src/mds/Resetter.cc +++ b/src/mds/Resetter.cc @@ -18,82 +18,21 @@ #include "mon/MonClient.h" #include "mds/events/EResetJournal.h" -Resetter::~Resetter() -{ -} -bool Resetter::ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, - bool force_new) +int Resetter::init(int rank) { - if (dest_type == CEPH_ENTITY_TYPE_MON) - return true; - - if (force_new) { - if (monc->wait_auth_rotating(10) < 0) - return false; + int r = MDSUtility::init(); + if (r < 0) { + return r; } - *authorizer = monc->auth->build_authorizer(dest_type); - return *authorizer != NULL; -} - -bool Resetter::ms_dispatch(Message *m) -{ - Mutex::Locker l(lock); - switch (m->get_type()) { - case CEPH_MSG_OSD_OPREPLY: - objecter->handle_osd_op_reply((MOSDOpReply *)m); - break; - case CEPH_MSG_OSD_MAP: - objecter->handle_osd_map((MOSDMap*)m); - break; - default: - return false; - } - return true; -} - - -void Resetter::init(int rank) -{ inodeno_t ino = MDS_INO_LOG_OFFSET + rank; - unsigned pg_pool = MDS_METADATA_POOL; - osdmap = new OSDMap(); - objecter = new Objecter(g_ceph_context, messenger, monc, osdmap, lock, timer, - 0, 0); - journaler = new Journaler(ino, pg_pool, CEPH_FS_ONDISK_MAGIC, - objecter, 0, 0, &timer); + journaler = new Journaler(ino, + mdsmap->get_metadata_pool(), + CEPH_FS_ONDISK_MAGIC, + objecter, 0, 0, &timer); - objecter->set_client_incarnation(0); - - messenger->add_dispatcher_head(this); - messenger->start(); - - monc->set_want_keys(CEPH_ENTITY_TYPE_MON|CEPH_ENTITY_TYPE_OSD|CEPH_ENTITY_TYPE_MDS); - monc->set_messenger(messenger); - monc->init(); - monc->authenticate(); - - client_t whoami = monc->get_global_id(); - messenger->set_myname(entity_name_t::CLIENT(whoami.v)); - - objecter->init_unlocked(); - lock.Lock(); - objecter->init_locked(); - lock.Unlock(); - objecter->wait_for_osd_map(); - timer.init(); -} - -void Resetter::shutdown() -{ - lock.Lock(); - timer.shutdown(); - objecter->shutdown_locked(); - lock.Unlock(); - objecter->shutdown_unlocked(); - messenger->shutdown(); - messenger->wait(); + return 0; } void Resetter::reset() @@ -172,5 +111,4 @@ void Resetter::reset() assert(r == 0); cout << "done" << std::endl; - shutdown(); } diff --git a/src/mds/Resetter.h b/src/mds/Resetter.h index b3bc0677740e..e967b246b1b4 100644 --- a/src/mds/Resetter.h +++ b/src/mds/Resetter.h @@ -14,54 +14,23 @@ #ifndef JOURNAL_RESETTER_H_ #define JOURNAL_RESETTER_H_ -#include "osd/OSDMap.h" -#include "osdc/Objecter.h" + #include "osdc/Journaler.h" -#include "msg/Dispatcher.h" -#include "msg/Messenger.h" -#include "auth/Auth.h" +#include "mds/MDSUtility.h" /** - * This class lets you dump out an mds journal for troubleshooting or whatever. + * This class lets you reset an mds journal for troubleshooting or whatever. * - * It was built to work with cmds so some of the design choices are random. - * To use, create a Resetter, call init(), and then call dump() with the name + * To use, create a Resetter, call init(), and then call reset() with the name * of the file to dump to. */ - -class Resetter : public Dispatcher { +class Resetter : public MDSUtility { public: - Objecter *objecter; Journaler *journaler; - OSDMap *osdmap; - Messenger *messenger; - MonClient *monc; - Mutex lock; - SafeTimer timer; - - /* - * The messenger should be a valid Messenger. You should call bind() - * before passing it in, but not do anything else. - * The MonClient needs to be valid, and you should have called - * build_initial_monmap(). - */ - Resetter(Messenger *messenger_, MonClient *monc_) : - Dispatcher(messenger_->cct), - messenger(messenger_), - monc(monc_), - lock("Resetter::lock"), - timer(g_ceph_context, lock) - {} - virtual ~Resetter(); + Resetter() : journaler(NULL) {} - bool ms_dispatch(Message *m); - bool ms_handle_reset(Connection *con) { return false; } - void ms_handle_remote_reset(Connection *con) {} - bool ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, - bool force_new); - void init(int rank); - void shutdown(); + int init(int rank); void reset(); };