From 740eea1ee88b8ecde7f67f31a39ba522f1c1d2eb Mon Sep 17 00:00:00 2001 From: Colin Patrick McCabe Date: Thu, 2 Jun 2011 15:50:22 -0700 Subject: [PATCH] Refactor MonClient, KeyRing MonClient should contain a KeyRing and a RotatingKeyRing. All the MonClient users, except possibly csyn, don't want to manage those objects themselves. Don't chdir until after we have opened the KeyRing. If the KeyRing is at a relative path, a chdir may make it inaccessible. Separate the chdir function from the daemonize function. Refactor the cmds argument parsing a little bit. Separate the special actions from the normal operations of the daemon. This should allow librados and libceph to support CephX finally! yay! Signed-off-by: Colin McCabe --- src/auth/KeyRing.cc | 10 +- src/auth/KeyRing.h | 2 + src/cfuse.cc | 2 +- src/cmds.cc | 282 +++++++++++++++++++++++--------------- src/cmon.cc | 1 + src/common/common_init.cc | 20 +-- src/common/common_init.h | 3 +- src/cosd.cc | 22 ++- src/csyn.cc | 4 +- src/libceph.cc | 2 +- src/librados.cc | 3 +- src/mon/MonClient.cc | 15 +- src/mon/MonClient.h | 15 +- src/tools/common.h | 1 + 14 files changed, 234 insertions(+), 148 deletions(-) diff --git a/src/auth/KeyRing.cc b/src/auth/KeyRing.cc index b9bd13583d7c8..2779cf2c3505e 100644 --- a/src/auth/KeyRing.cc +++ b/src/auth/KeyRing.cc @@ -39,8 +39,8 @@ from_ceph_conf(const md_config_t *conf) auto_ptr < KeyRing > keyring(new KeyRing()); if (!is_supported_auth(CEPH_AUTH_CEPHX)) { - derr << "KeyRing::from_ceph_conf: CephX auth is not supported." << dendl; - return NULL; + dout(2) << "KeyRing::from_ceph_conf: CephX auth is not supported." << dendl; + return keyring.release(); } int ret = 0; @@ -89,6 +89,12 @@ from_ceph_conf(const md_config_t *conf) return keyring.release(); } +KeyRing *KeyRing:: +create_empty() +{ + return new KeyRing(); +} + int KeyRing::set_modifier(const char *type, const char *val, EntityName& name, map& caps) { if (!val) diff --git a/src/auth/KeyRing.h b/src/auth/KeyRing.h index 9869875b7bb06..a3e26e2b6c1d2 100644 --- a/src/auth/KeyRing.h +++ b/src/auth/KeyRing.h @@ -30,6 +30,8 @@ class KeyRing { public: /* Create a KeyRing from a Ceph configuration */ static KeyRing *from_ceph_conf(const md_config_t *conf); + /* Create an empty KeyRing */ + static KeyRing *create_empty(); map& get_keys() { return keys; } // yuck diff --git a/src/cfuse.cc b/src/cfuse.cc index d1fa3f2bcc33d..7fbbf6e8cedf7 100644 --- a/src/cfuse.cc +++ b/src/cfuse.cc @@ -81,7 +81,7 @@ int main(int argc, const char **argv, const char *envp[]) { } // get monmap - MonClient mc; + MonClient mc(&g_ceph_context); int ret = mc.build_initial_monmap(); if (ret == -EINVAL) usage(); diff --git a/src/cmds.cc b/src/cmds.cc index 39fc871cdc576..664b2b61a0aa5 100644 --- a/src/cmds.cc +++ b/src/cmds.cc @@ -22,6 +22,7 @@ using namespace std; #include "common/config.h" +#include "common/strtol.h" #include "mon/MonMap.h" #include "mds/MDS.h" @@ -58,6 +59,65 @@ void usage() generic_server_usage(); } +static int do_cmds_special_action(const std::string &action, + const std::string &dump_file, int rank) +{ + SimpleMessenger *messenger = new SimpleMessenger(); + messenger->bind(getpid()); + MonClient mc(&g_ceph_context); + if (mc.build_initial_monmap() < 0) + return -1; + + if (action == "dump-journal") { + dout(0) << "dumping journal for mds" << rank << " to " << dump_file << dendl; + Dumper *journal_dumper = new Dumper(messenger, &mc); + journal_dumper->init(rank); + journal_dumper->dump(dump_file.c_str()); + mc.shutdown(); + } + else if (action == "undump-journal") { + dout(0) << "undumping journal for mds" << rank << " from " << dump_file << dendl; + Dumper *journal_dumper = new Dumper(messenger, &mc); + journal_dumper->init(rank); + journal_dumper->undump(dump_file.c_str()); + mc.shutdown(); + } + else if (action == "reset-journal") { + dout(0) << "resetting journal" << dendl; + Resetter *jr = new Resetter(messenger, &mc); + jr->init(rank); + jr->reset(); + mc.shutdown(); + } + else { + assert(0); + } + return 0; +} + +static void set_special_action(std::string &dest, const std::string &act) +{ + if (!dest.empty()) { + derr << "Parse error! Can't specify more than one action. You " + << "specified both " << act << " and " << dest << "\n" << dendl; + usage(); + exit(1); + } + dest = act; +} + +static int parse_rank(const char *opt_name, const std::string &val) +{ + std::string err; + int ret = strict_strtol(val.c_str(), 10, &err); + if (!err.empty()) { + derr << "error parsing " << opt_name << ": failed to parse rank. " + << "It must be an int." << "\n" << dendl; + usage(); + } + return ret; +} + int main(int argc, const char **argv) { DEFINE_CONF_VARS(usage); @@ -66,146 +126,140 @@ int main(int argc, const char **argv) env_to_vec(args); common_init(args, CEPH_ENTITY_TYPE_MDS, CODE_ENVIRONMENT_DAEMON, 0); - KeyRing *cmds_keyring = KeyRing::from_ceph_conf(g_ceph_context._conf); - if (!cmds_keyring) { - derr << "Unable to get a Ceph keyring." << dendl; - return 1; - } // mds specific args int shadow = 0; - int dump_journal = -1; - int undump_journal = -1; - const char *dump_file = NULL; - int reset_journal = -1; - FOR_EACH_ARG(args) { - if (CEPH_ARGPARSE_EQ("dump-journal", '\0')) { - CEPH_ARGPARSE_SET_ARG_VAL(&dump_journal, OPT_INT); - CEPH_ARGPARSE_SET_ARG_VAL(&dump_file, OPT_STR); - cout << "dumping journal for mds" << dump_journal << " to " << dump_file << std::endl; - } else if (CEPH_ARGPARSE_EQ("undump-journal", '\0')) { - CEPH_ARGPARSE_SET_ARG_VAL(&undump_journal, OPT_INT); - CEPH_ARGPARSE_SET_ARG_VAL(&dump_file, OPT_STR); - cout << "undumping journal for mds" << dump_journal << " to " << dump_file << std::endl; - } else if (CEPH_ARGPARSE_EQ("reset-journal", '\0')) { - CEPH_ARGPARSE_SET_ARG_VAL(&reset_journal, OPT_INT); - } else if (CEPH_ARGPARSE_EQ("journal-check", '\0')) { - int check_rank; - CEPH_ARGPARSE_SET_ARG_VAL(&check_rank, OPT_INT); - + int rank = -1; + std::string dump_file; + + std::string val, action; + for (std::vector::iterator i = args.begin(); i != args.end(); ) { + if (ceph_argparse_witharg(args, i, &val, "--dump-journal", (char*)NULL)) { + set_special_action(action, "dump-journal"); + rank = parse_rank("dump-journal", val); + if (i == args.end()) { + derr << "error parsing --dump-journal: you must give a second " + << "dump-journal argument: the filename to dump the journal to. " + << "\n" << dendl; + usage(); + } + dump_file = *i++; + } + else if (ceph_argparse_witharg(args, i, &val, "--undump-journal", (char*)NULL)) { + set_special_action(action, "undump-journal"); + rank = parse_rank("undump-journal", val); + if (i == args.end()) { + derr << "error parsing --undump-journal: you must give a second " + << "undump-journal argument: the filename to undump the journal from. " + << "\n" << dendl; + usage(); + } + dump_file = *i++; + } + else if (ceph_argparse_witharg(args, i, &val, "--reset-journal", (char*)NULL)) { + set_special_action(action, "reset-journal"); + rank = parse_rank("reset-journal", val); + } + else if (ceph_argparse_witharg(args, i, &val, "--journal-check", (char*)NULL)) { + int r = parse_rank("journal-check", val); if (shadow) { dout(0) << "Error: can only select one standby state" << dendl; return -1; } - dout(0) << "requesting oneshot_replay for mds" << check_rank << dendl; + dout(0) << "requesting oneshot_replay for mds" << r << dendl; shadow = MDSMap::STATE_ONESHOT_REPLAY; - g_conf.mds_standby_for_rank = check_rank; - ++i; - } else if (CEPH_ARGPARSE_EQ("hot-standby", '\0')) { - int check_rank; - CEPH_ARGPARSE_SET_ARG_VAL(&check_rank, OPT_INT); + g_conf.mds_standby_for_rank = r; + } + else if (ceph_argparse_witharg(args, i, &val, "--hot-standby", (char*)NULL)) { + int r = parse_rank("hot-standby", val); if (shadow) { dout(0) << "Error: can only select one standby state" << dendl; return -1; } - dout(0) << "requesting standby_replay for mds" << check_rank << dendl; + dout(0) << "requesting standby_replay for mds" << r << dendl; shadow = MDSMap::STATE_STANDBY_REPLAY; - g_conf.mds_standby_for_rank = check_rank; - } else { - derr << "unrecognized arg " << args[i] << dendl; + g_conf.mds_standby_for_rank = r; + } + else { + derr << "Error: can't understand argument: " << *i << "\n" << dendl; usage(); } } - if (g_conf.name.has_default_id() && dump_journal < 0 && reset_journal < 0) { + + // Check for special actions + if (!action.empty()) { + return do_cmds_special_action(action, dump_file, rank); + } + + // Normal startup + if (g_conf.name.has_default_id()) { derr << "must specify '-i name' with the cmds instance name" << dendl; usage(); } + SimpleMessenger *messenger = new SimpleMessenger(); + messenger->bind(getpid()); + cout << "starting " << g_conf.name << " at " << messenger->get_ms_addr() + << std::endl; + messenger->register_entity(entity_name_t::MDS(-1)); + uint64_t supported = + CEPH_FEATURE_UID | + CEPH_FEATURE_NOSRCADDR | + CEPH_FEATURE_DIRLAYOUTHASH; + messenger->set_default_policy(SimpleMessenger::Policy::client(supported, 0)); + messenger->set_policy(entity_name_t::TYPE_MON, + SimpleMessenger::Policy::client(supported, + CEPH_FEATURE_UID)); + messenger->set_policy(entity_name_t::TYPE_MDS, + SimpleMessenger::Policy::lossless_peer(supported, + CEPH_FEATURE_UID)); + messenger->set_policy(entity_name_t::TYPE_CLIENT, + SimpleMessenger::Policy::stateful_server(supported, 0)); + + if (shadow != MDSMap::STATE_ONESHOT_REPLAY) + common_init_daemonize(&g_ceph_context, 0); + common_init_finish(&g_ceph_context); + // get monmap - RotatingKeyRing rkeys(CEPH_ENTITY_TYPE_MDS, cmds_keyring); - MonClient mc(&rkeys); + MonClient mc(&g_ceph_context); if (mc.build_initial_monmap() < 0) return -1; + common_init_chdir(&g_ceph_context); - SimpleMessenger *messenger = new SimpleMessenger(); - messenger->bind(getpid()); - if (dump_journal >= 0) { - Dumper *journal_dumper = new Dumper(messenger, &mc); - journal_dumper->init(dump_journal); - journal_dumper->dump(dump_file); - mc.shutdown(); - } else if (undump_journal >= 0) { - Dumper *journal_dumper = new Dumper(messenger, &mc); - journal_dumper->init(undump_journal); - journal_dumper->undump(dump_file); - mc.shutdown(); - } else if (reset_journal >= 0) { - Resetter *jr = new Resetter(messenger, &mc); - jr->init(reset_journal); - jr->reset(); - mc.shutdown(); - } else { - cout << "starting " << g_conf.name << " at " << messenger->get_ms_addr() - << std::endl; - - messenger->register_entity(entity_name_t::MDS(-1)); - assert_warn(messenger); - if (!messenger) - return 1; - - uint64_t supported = - CEPH_FEATURE_UID | - CEPH_FEATURE_NOSRCADDR | - CEPH_FEATURE_DIRLAYOUTHASH; - messenger->set_default_policy(SimpleMessenger::Policy::client(supported, 0)); - messenger->set_policy(entity_name_t::TYPE_MON, - SimpleMessenger::Policy::client(supported, - CEPH_FEATURE_UID)); - messenger->set_policy(entity_name_t::TYPE_MDS, - SimpleMessenger::Policy::lossless_peer(supported, - CEPH_FEATURE_UID)); - messenger->set_policy(entity_name_t::TYPE_CLIENT, - SimpleMessenger::Policy::stateful_server(supported, 0)); - - if (shadow != MDSMap::STATE_ONESHOT_REPLAY) - common_init_daemonize(&g_ceph_context, 0); - common_init_finish(&g_ceph_context); - ... - messenger->start(); - - // start mds - MDS *mds = new MDS(g_conf.name.get_id().c_str(), messenger, &mc); - - // in case we have to respawn... - mds->orig_argc = argc; - mds->orig_argv = argv; - - if (shadow) - mds->init(shadow); - else - mds->init(); - - messenger->wait(); - - // yuck: grab the mds lock, so we can be sure that whoever in *mds - // called shutdown finishes what they were doing. - mds->mds_lock.Lock(); - mds->mds_lock.Unlock(); - - // only delete if it was a clean shutdown (to aid memory leak - // detection, etc.). don't bother if it was a suicide. - if (mds->is_stopped()) - delete mds; - - // cd on exit, so that gmon.out (if any) goes into a separate directory for each node. - char s[20]; - snprintf(s, sizeof(s), "gmon/%d", getpid()); - if ((mkdir(s, 0755) == 0) && (chdir(s) == 0)) { - dout(0) << "cmds: gmon.out should be in " << s << dendl; - } + messenger->start(); - generic_dout(0) << "stopped." << dendl; + // start mds + MDS *mds = new MDS(g_conf.name.get_id().c_str(), messenger, &mc); + + // in case we have to respawn... + mds->orig_argc = argc; + mds->orig_argv = argv; + + if (shadow) + mds->init(shadow); + else + mds->init(); + + messenger->wait(); + + // yuck: grab the mds lock, so we can be sure that whoever in *mds + // called shutdown finishes what they were doing. + mds->mds_lock.Lock(); + mds->mds_lock.Unlock(); + + // only delete if it was a clean shutdown (to aid memory leak + // detection, etc.). don't bother if it was a suicide. + if (mds->is_stopped()) + delete mds; + + // cd on exit, so that gmon.out (if any) goes into a separate directory for each node. + char s[20]; + snprintf(s, sizeof(s), "gmon/%d", getpid()); + if ((mkdir(s, 0755) == 0) && (chdir(s) == 0)) { + dout(0) << "cmds: gmon.out should be in " << s << dendl; } + + generic_dout(0) << "stopped." << dendl; return 0; } diff --git a/src/cmon.cc b/src/cmon.cc index 4b616da750d3f..86f7c5532e4ea 100644 --- a/src/cmon.cc +++ b/src/cmon.cc @@ -261,6 +261,7 @@ int main(int argc, const char **argv) common_init_daemonize(&g_ceph_context, 0); common_init_finish(&g_ceph_context); + common_init_chdir(&g_ceph_context); messenger->start(); uint64_t supported = diff --git a/src/common/common_init.cc b/src/common/common_init.cc index 83cfbc99cb282..a1124e8bdf915 100644 --- a/src/common/common_init.cc +++ b/src/common/common_init.cc @@ -190,14 +190,6 @@ void common_init_daemonize(const CephContext *cct, int flags) exit(1); } - if (!conf->chdir.empty()) { - if (::chdir(conf->chdir.c_str())) { - int err = errno; - derr << "common_init_daemonize: failed to chdir to directory: '" - << conf->chdir << "': " << cpp_strerror(err) << dendl; - } - } - if (atexit(pidfile_remove_void)) { derr << "common_init_daemonize: failed to set pidfile_remove function " << "to run at exit." << dendl; @@ -243,6 +235,18 @@ void common_init_daemonize(const CephContext *cct, int flags) dout(1) << "finished common_init_daemonize" << dendl; } +void common_init_chdir(const CephContext *cct) +{ + const md_config_t *conf = cct->_conf; + if (conf->chdir.empty()) + return; + if (::chdir(conf->chdir.c_str())) { + int err = errno; + derr << "common_init_chdir: failed to chdir to directory: '" + << conf->chdir << "': " << cpp_strerror(err) << dendl; + } +} + /* Please be sure that this can safely be called multiple times by the * same application. */ void common_init_finish(CephContext *cct) diff --git a/src/common/common_init.h b/src/common/common_init.h index 7459f79c437fd..ff20b9128b48d 100644 --- a/src/common/common_init.h +++ b/src/common/common_init.h @@ -18,7 +18,7 @@ enum common_init_flags_t { // By default, don't read a configuration file CINIT_FLAG_NO_DEFAULT_CONFIG_FILE = 0x2, - // Don't close stderr + // Don't close stderr (in daemonize) CINIT_FLAG_NO_CLOSE_STDERR = 0x4, }; @@ -32,5 +32,6 @@ int common_init_shutdown_stderr(); void common_init_daemonize(const CephContext *cct, int flags); int common_init_shutdown_stderr(void); void common_init_finish(CephContext *cct); +void common_init_chdir(const CephContext *cct); #endif diff --git a/src/cosd.cc b/src/cosd.cc index 01a0ceff70298..ab863b8693b6f 100644 --- a/src/cosd.cc +++ b/src/cosd.cc @@ -57,12 +57,6 @@ int main(int argc, const char **argv) vector::iterator args_iter; common_init(args, CEPH_ENTITY_TYPE_OSD, CODE_ENVIRONMENT_DAEMON, 0); - KeyRing *cosd_keyring = KeyRing::from_ceph_conf(g_ceph_context._conf); - if (!cosd_keyring) { - derr << "Unable to get a Ceph keyring." << dendl; - return 1; - } - ceph_heap_profiler_init(); // osd specific args @@ -128,8 +122,7 @@ int main(int argc, const char **argv) if (mkfs) { common_init_finish(&g_ceph_context); - RotatingKeyRing rkeys(CEPH_ENTITY_TYPE_OSD, cosd_keyring); - MonClient mc(&rkeys); + MonClient mc(&g_ceph_context); if (mc.build_initial_monmap() < 0) return -1; if (mc.get_monmap_privately() < 0) @@ -148,12 +141,17 @@ int main(int argc, const char **argv) } if (mkkey) { common_init_finish(&g_ceph_context); + KeyRing *keyring = KeyRing::create_empty(); + if (!keyring) { + derr << "Unable to get a Ceph keyring." << dendl; + return 1; + } EntityName ename(g_conf.name); EntityAuth eauth; eauth.key.create(CEPH_CRYPTO_AES); - cosd_keyring->add(ename, eauth); + keyring->add(ename, eauth); bufferlist bl; - cosd_keyring->encode_plaintext(bl); + keyring->encode_plaintext(bl); int r = bl.write_file(g_conf.keyring.c_str(), 0600); if (r) derr << TEXT_RED << " ** ERROR: writing new keyring to " << g_conf.keyring @@ -279,10 +277,10 @@ int main(int argc, const char **argv) // Leave stderr open in case we need to report errors. common_init_daemonize(&g_ceph_context, CINIT_FLAG_NO_CLOSE_STDERR); common_init_finish(&g_ceph_context); - RotatingKeyRing rkeys(CEPH_ENTITY_TYPE_OSD, cosd_keyring); - MonClient mc(&rkeys); + MonClient mc(&g_ceph_context); if (mc.build_initial_monmap() < 0) return -1; + common_init_chdir(&g_ceph_context); OSD *osd = new OSD(whoami, cluster_messenger, client_messenger, messenger_hb, &mc, g_conf.osd_data, g_conf.osd_journal); diff --git a/src/csyn.cc b/src/csyn.cc index 1b7c29bf0c6e9..c654cc3a962c9 100644 --- a/src/csyn.cc +++ b/src/csyn.cc @@ -54,7 +54,7 @@ int main(int argc, const char **argv, char *envp[]) vec_to_argv(args, argc, argv); // get monmap - MonClient mc; + MonClient mc(&g_ceph_context); if (mc.build_initial_monmap() < 0) return -1; @@ -68,7 +68,7 @@ int main(int argc, const char **argv, char *envp[]) messengers[i] = new SimpleMessenger(); messengers[i]->register_entity(entity_name_t(entity_name_t::TYPE_CLIENT,-1)); messengers[i]->bind(i * 1000000 + getpid()); - mclients[i] = new MonClient(); + mclients[i] = new MonClient(&g_ceph_context); mclients[i]->build_initial_monmap(); Client *client = new Client(messengers[i], mclients[i]); client->set_filer_flags(syn_filer_flags); diff --git a/src/libceph.cc b/src/libceph.cc index e5b6782337587..14e512d6155f5 100644 --- a/src/libceph.cc +++ b/src/libceph.cc @@ -73,7 +73,7 @@ public: common_init_finish(cct); //monmap - monclient = new MonClient(); + monclient = new MonClient(cct); if (monclient->build_initial_monmap() < 0) { shutdown(); return -1000; diff --git a/src/librados.cc b/src/librados.cc index 7b3ff3ca043d1..ad19aefc00a83 100644 --- a/src/librados.cc +++ b/src/librados.cc @@ -404,7 +404,8 @@ private: public: RadosClient(CephContext *cct_) : cct(cct_), conf(cct_->_conf), - state(DISCONNECTED), messenger(NULL), objecter(NULL), + state(DISCONNECTED), monclient(cct_), + messenger(NULL), objecter(NULL), lock("radosclient"), timer(lock), max_watch_cookie(0) { } diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc index ad77d05585a6a..6554be7c879a8 100644 --- a/src/mon/MonClient.cc +++ b/src/mon/MonClient.cc @@ -43,8 +43,9 @@ MonClient::~MonClient() { - if (auth) - delete auth; + delete auth; + delete keyring; + delete rotating_secrets; } /* @@ -316,12 +317,19 @@ void MonClient::handle_monmap(MMonMap *m) // ---------------------- -void MonClient::init() +int MonClient::init() { dout(10) << "init" << dendl; messenger->add_dispatcher_head(this); + keyring = KeyRing::from_ceph_conf(cct->_conf); + if (!keyring) { + derr << "MonClient::init(): Failed to create keyring" << dendl; + return -EDOM; + } + rotating_secrets = new RotatingKeyRing(CEPH_ENTITY_TYPE_OSD, keyring); + entity_name = g_conf.name; Mutex::Locker l(monc_lock); @@ -346,6 +354,7 @@ void MonClient::init() dout(0) << "WARNING: unknown auth protocol defined: " << *iter << dendl; } } + return 0; } void MonClient::shutdown() diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h index 5cb3daf14822a..c2b83ab25fb16 100644 --- a/src/mon/MonClient.h +++ b/src/mon/MonClient.h @@ -47,6 +47,7 @@ class MonClient : public Dispatcher { public: MonMap monmap; private: + CephContext *cct; MonClientState state; Messenger *messenger; @@ -161,10 +162,12 @@ public: _sub_got(what, have); } + KeyRing *keyring; RotatingKeyRing *rotating_secrets; public: - MonClient(RotatingKeyRing *rkeys=0) : + MonClient(CephContext *cct_) : + cct(cct_), state(MC_STATE_NONE), messenger(NULL), cur_con(NULL), @@ -176,11 +179,14 @@ public: want_keys(0), global_id(0), authenticate_err(0), auth(NULL), - rotating_secrets(rkeys) { } + keyring(NULL), + rotating_secrets(NULL) + { + } ~MonClient(); - void init(); + int init(); void shutdown(); void set_log_client(LogClient *clog) { @@ -248,6 +254,9 @@ public: if (auth) auth->add_want_keys(want); } +private: + MonClient(const MonClient &rhs); + MonClient& operator=(const MonClient &rhs); }; #endif diff --git a/src/tools/common.h b/src/tools/common.h index 3a4a47cb17e5e..46e310fdfb3c5 100644 --- a/src/tools/common.h +++ b/src/tools/common.h @@ -53,6 +53,7 @@ struct ceph_tool_data bool concise; ceph_tool_data() : + mc(&g_ceph_context), updates(EVERYTHING_UPDATE), log(&std::cout), slog(NULL), -- 2.39.5