From: Samuel Just Date: Thu, 21 Mar 2013 18:19:45 +0000 (-0700) Subject: OSD: notify mon prior to shutdown X-Git-Tag: v0.62~191^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=7bfaaf7a20252754bcf239aab6582c0ad3ea9763;p=ceph.git OSD: notify mon prior to shutdown Signed-off-by: Samuel Just --- diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 52880f59feba..f012091f870d 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -436,6 +436,9 @@ OPTION(osd_failsafe_nearfull_ratio, OPT_FLOAT, .90) // what % full makes an OSD OPTION(osd_client_op_priority, OPT_INT, 63) OPTION(osd_recovery_op_priority, OPT_INT, 10) +// Max time to wait between notifying mon of shutdown and shutting down +OPTION(osd_mon_shutdown_timeout, OPT_DOUBLE, 5) + OPTION(filestore, OPT_BOOL, false) // Tests index failure paths diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 0163429afb61..978c24056f5b 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -55,6 +55,7 @@ #include "messages/MPing.h" #include "messages/MOSDPing.h" #include "messages/MOSDFailure.h" +#include "messages/MOSDMarkMeDown.h" #include "messages/MOSDOp.h" #include "messages/MOSDOpReply.h" #include "messages/MOSDSubOp.h" @@ -184,7 +185,9 @@ OSDService::OSDService(OSD *osd) : in_progress_split_lock("OSDService::in_progress_split_lock"), full_status_lock("OSDService::full_status_lock"), cur_state(NONE), - last_msg(0) + last_msg(0), + is_stopping_lock("OSDService::is_stopping_lock"), + state(NOT_STOPPING) {} void OSDService::_start_split(const set &pgs) @@ -1182,6 +1185,8 @@ void OSD::suicide(int exitcode) int OSD::shutdown() { + if (!service.prepare_to_stop()) + return 0; // already shutting down osd_lock.Lock(); if (is_stopping()) { osd_lock.Unlock(); @@ -3398,6 +3403,11 @@ bool OSD::heartbeat_dispatch(Message *m) bool OSD::ms_dispatch(Message *m) { + if (m->get_type() == MSG_OSD_MARK_ME_DOWN) { + service.got_stop_ack(); + return true; + } + // lock! osd_lock.Lock(); @@ -3858,6 +3868,37 @@ void OSDService::dec_scrubs_active() sched_scrub_lock.Unlock(); } +bool OSDService::prepare_to_stop() { + Mutex::Locker l(is_stopping_lock); + if (state != NOT_STOPPING) + return false; + + state = PREPARING_TO_STOP; + monc->send_mon_message( + new MOSDMarkMeDown( + monc->get_fsid(), + get_osdmap()->get_inst(whoami), + get_osdmap()->get_epoch(), + false + )); + utime_t now = ceph_clock_now(g_ceph_context); + utime_t timeout; + timeout.set_from_double( + now + g_conf->osd_mon_shutdown_timeout); + while ((ceph_clock_now(g_ceph_context) < timeout) && + (state != STOPPING)) { + is_stopping_cond.WaitUntil(is_stopping_lock, timeout); + } + state = STOPPING; + return true; +} + +void OSDService::got_stop_ack() { + Mutex::Locker l(is_stopping_lock); + dout(10) << "Got stop ack" << dendl; + state = STOPPING; + is_stopping_cond.Signal(); +} // ===================================================== // MAP @@ -4104,9 +4145,14 @@ void OSD::handle_osd_map(MOSDMap *m) !osdmap->get_addr(whoami).probably_equals(client_messenger->get_myaddr()) || !osdmap->get_cluster_addr(whoami).probably_equals(cluster_messenger->get_myaddr()) || !osdmap->get_hb_addr(whoami).probably_equals(hbserver_messenger->get_myaddr())) { - if (!osdmap->is_up(whoami)) - clog.warn() << "map e" << osdmap->get_epoch() - << " wrongly marked me down"; + if (!osdmap->is_up(whoami)) { + if (service.is_preparing_to_stop()) { + service.got_stop_ack(); + } else { + clog.warn() << "map e" << osdmap->get_epoch() + << " wrongly marked me down"; + } + } else if (!osdmap->get_addr(whoami).probably_equals(client_messenger->get_myaddr())) clog.error() << "map e" << osdmap->get_epoch() << " had wrong client addr (" << osdmap->get_addr(whoami) @@ -4120,25 +4166,27 @@ void OSD::handle_osd_map(MOSDMap *m) << " had wrong hb addr (" << osdmap->get_hb_addr(whoami) << " != my " << hbserver_messenger->get_myaddr() << ")"; - state = STATE_BOOTING; - up_epoch = 0; - do_restart = true; - bind_epoch = osdmap->get_epoch(); + if (!service.is_stopping()) { + state = STATE_BOOTING; + up_epoch = 0; + do_restart = true; + bind_epoch = osdmap->get_epoch(); - int cport = cluster_messenger->get_myaddr().get_port(); - int hbport = hbserver_messenger->get_myaddr().get_port(); + int cport = cluster_messenger->get_myaddr().get_port(); + int hbport = hbserver_messenger->get_myaddr().get_port(); - int r = cluster_messenger->rebind(hbport); - if (r != 0) - do_shutdown = true; // FIXME: do_restart? + int r = cluster_messenger->rebind(hbport); + if (r != 0) + do_shutdown = true; // FIXME: do_restart? - r = hbserver_messenger->rebind(cport); - if (r != 0) - do_shutdown = true; // FIXME: do_restart? + r = hbserver_messenger->rebind(cport); + if (r != 0) + do_shutdown = true; // FIXME: do_restart? - hbclient_messenger->mark_down_all(); + hbclient_messenger->mark_down_all(); - reset_heartbeat_peers(); + reset_heartbeat_peers(); + } } } diff --git a/src/osd/OSD.h b/src/osd/OSD.h index d41c9fec4d6f..148b761f5324 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -406,6 +406,24 @@ public: void check_nearfull_warning(const osd_stat_t &stat); bool check_failsafe_full(); + // -- stopping -- + Mutex is_stopping_lock; + Cond is_stopping_cond; + enum { + NOT_STOPPING, + PREPARING_TO_STOP, + STOPPING } state; + bool is_stopping() { + Mutex::Locker l(is_stopping_lock); + return state == STOPPING; + } + bool is_preparing_to_stop() { + Mutex::Locker l(is_stopping_lock); + return state == PREPARING_TO_STOP; + } + bool prepare_to_stop(); + void got_stop_ack(); + OSDService(OSD *osd); }; class OSD : public Dispatcher,