From 668f3db140476bd987d6f9b61479890e7be02500 Mon Sep 17 00:00:00 2001 From: xie xingguo Date: Thu, 30 May 2019 14:42:23 +0800 Subject: [PATCH] osd: unify sources of no{up,down,in,out} flags into singleton helpers Currently there are various ways to set no{up,down,in,out} flags, by osd, by crush node, or by device class. Use traditional is_no{up,down,in,out} helper to catch-all, and add special wrappers for certain internal consumers. This way osd won't ignore the noup flag set by the new crush node, device class, or any future sources. Signed-off-by: xie xingguo (cherry picked from commit 9d11a3a1c96edd4f38f4584ce76febf920763883) Conflicts: slight conflict from the "Remove dependence on 'using namespac'" change, see https://github.com/ceph/ceph/pull/27255 --- src/crimson/osd/osd.cc | 2 +- src/mon/OSDMonitor.cc | 99 ++++-------------------------------------- src/osd/OSD.cc | 6 +-- src/osd/OSDMap.h | 88 ++++++++++++++++++++----------------- 4 files changed, 59 insertions(+), 136 deletions(-) diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc index 1d0dbf89d7028..bdb1642e7d2e6 100644 --- a/src/crimson/osd/osd.cc +++ b/src/crimson/osd/osd.cc @@ -238,7 +238,7 @@ seastar::future<> OSD::_preboot(version_t oldest, version_t newest) if (osdmap->get_epoch() > newest - 1) { throw std::runtime_error("i am destroyed"); } - } else if (osdmap->test_flag(CEPH_OSDMAP_NOUP) || osdmap->is_noup(whoami)) { + } else if (osdmap->is_noup(whoami)) { logger().warn("osdmap NOUP flag is set, waiting for it to clear"); } else if (!osdmap->test_flag(CEPH_OSDMAP_SORTBITWISE)) { logger().error("osdmap SORTBITWISE OSDMap flag is NOT set; please set it"); diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index d4270b7d8bccc..b9eed198a4d17 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -2441,33 +2441,12 @@ bool OSDMonitor::prepare_mark_me_down(MonOpRequestRef op) bool OSDMonitor::can_mark_down(int i) { - if (osdmap.test_flag(CEPH_OSDMAP_NODOWN)) { - dout(5) << __func__ << " NODOWN flag set, will not mark osd." << i - << " down" << dendl; - return false; - } - if (osdmap.is_nodown(i)) { dout(5) << __func__ << " osd." << i << " is marked as nodown, " << "will not mark it down" << dendl; return false; } - if (osdmap.get_osd_crush_node_flags(i) & CEPH_OSD_NODOWN) { - dout(5) << __func__ << " osd." << i - << " is marked as nodown via a crush node flag, " - << "will not mark it down" << dendl; - return false; - } - - if (auto class_id = osdmap.crush->get_item_class_id(i); class_id >= 0 && - (osdmap.get_device_class_flags(class_id) & CEPH_OSD_NODOWN)) { - dout(5) << __func__ << " osd." << i - << " is marked as nodown via device class, " - << "will not mark it down" << dendl; - return false; - } - int num_osds = osdmap.get_num_osds(); if (num_osds == 0) { dout(5) << __func__ << " no osds" << dendl; @@ -2486,33 +2465,12 @@ bool OSDMonitor::can_mark_down(int i) bool OSDMonitor::can_mark_up(int i) { - if (osdmap.test_flag(CEPH_OSDMAP_NOUP)) { - dout(5) << __func__ << " NOUP flag set, will not mark osd." << i - << " up" << dendl; - return false; - } - if (osdmap.is_noup(i)) { dout(5) << __func__ << " osd." << i << " is marked as noup, " << "will not mark it up" << dendl; return false; } - if (osdmap.get_osd_crush_node_flags(i) & CEPH_OSD_NOUP) { - dout(5) << __func__ << " osd." << i - << " is marked as noup via a crush node flag, " - << "will not mark it up" << dendl; - return false; - } - - if (auto class_id = osdmap.crush->get_item_class_id(i); class_id >= 0 && - (osdmap.get_device_class_flags(class_id) & CEPH_OSD_NOUP)) { - dout(5) << __func__ << " osd." << i - << " is marked as noup via device class, " - << "will not mark it up" << dendl; - return false; - } - return true; } @@ -2522,32 +2480,12 @@ bool OSDMonitor::can_mark_up(int i) */ bool OSDMonitor::can_mark_out(int i) { - if (osdmap.test_flag(CEPH_OSDMAP_NOOUT)) { - dout(5) << __func__ << " NOOUT flag set, will not mark osds out" << dendl; - return false; - } - if (osdmap.is_noout(i)) { dout(5) << __func__ << " osd." << i << " is marked as noout, " << "will not mark it out" << dendl; return false; } - if (osdmap.get_osd_crush_node_flags(i) & CEPH_OSD_NOOUT) { - dout(5) << __func__ << " osd." << i - << " is marked as noout via a crush node flag, " - << "will not mark it out" << dendl; - return false; - } - - if (auto class_id = osdmap.crush->get_item_class_id(i); class_id >= 0 && - (osdmap.get_device_class_flags(class_id) & CEPH_OSD_NOOUT)) { - dout(5) << __func__ << " osd." << i - << " is marked as noout via device class, " - << "will not mark it out" << dendl; - return false; - } - int num_osds = osdmap.get_num_osds(); if (num_osds == 0) { dout(5) << __func__ << " no osds" << dendl; @@ -2572,33 +2510,12 @@ bool OSDMonitor::can_mark_out(int i) bool OSDMonitor::can_mark_in(int i) { - if (osdmap.test_flag(CEPH_OSDMAP_NOIN)) { - dout(5) << __func__ << " NOIN flag set, will not mark osd." << i - << " in" << dendl; - return false; - } - if (osdmap.is_noin(i)) { dout(5) << __func__ << " osd." << i << " is marked as noin, " << "will not mark it in" << dendl; return false; } - if (osdmap.get_osd_crush_node_flags(i) & CEPH_OSD_NOIN) { - dout(5) << __func__ << " osd." << i - << " is marked as noin via a crush node flag, " - << "will not mark it in" << dendl; - return false; - } - - if (auto class_id = osdmap.crush->get_item_class_id(i); class_id >= 0 && - (osdmap.get_device_class_flags(class_id) & CEPH_OSD_NOIN)) { - dout(5) << __func__ << " osd." << i - << " is marked as noin via device class, " - << "will not mark it in" << dendl; - return false; - } - return true; } @@ -10656,43 +10573,43 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op, } if (do_set) { if (flags & CEPH_OSD_NOUP) { - any |= osdmap.is_noup(osd) ? + any |= osdmap.is_noup_by_osd(osd) ? pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOUP) : pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOUP); } if (flags & CEPH_OSD_NODOWN) { - any |= osdmap.is_nodown(osd) ? + any |= osdmap.is_nodown_by_osd(osd) ? pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NODOWN) : pending_inc.pending_osd_state_set(osd, CEPH_OSD_NODOWN); } if (flags & CEPH_OSD_NOIN) { - any |= osdmap.is_noin(osd) ? + any |= osdmap.is_noin_by_osd(osd) ? pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOIN) : pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOIN); } if (flags & CEPH_OSD_NOOUT) { - any |= osdmap.is_noout(osd) ? + any |= osdmap.is_noout_by_osd(osd) ? pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOOUT) : pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOOUT); } } else { if (flags & CEPH_OSD_NOUP) { - any |= osdmap.is_noup(osd) ? + any |= osdmap.is_noup_by_osd(osd) ? pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOUP) : pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOUP); } if (flags & CEPH_OSD_NODOWN) { - any |= osdmap.is_nodown(osd) ? + any |= osdmap.is_nodown_by_osd(osd) ? pending_inc.pending_osd_state_set(osd, CEPH_OSD_NODOWN) : pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NODOWN); } if (flags & CEPH_OSD_NOIN) { - any |= osdmap.is_noin(osd) ? + any |= osdmap.is_noin_by_osd(osd) ? pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOIN) : pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOIN); } if (flags & CEPH_OSD_NOOUT) { - any |= osdmap.is_noout(osd) ? + any |= osdmap.is_noout_by_osd(osd) ? pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOOUT) : pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOOUT); } diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 09657911377cb..7d4474460b0ad 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -5882,7 +5882,7 @@ void OSD::_preboot(epoch_t oldest, epoch_t newest) if (osdmap->get_epoch() > newest - 1) { exit(0); } - } else if (osdmap->test_flag(CEPH_OSDMAP_NOUP) || osdmap->is_noup(whoami)) { + } else if (osdmap->is_noup(whoami)) { derr << "osdmap NOUP flag is set, waiting for it to clear" << dendl; } else if (!osdmap->test_flag(CEPH_OSDMAP_SORTBITWISE)) { derr << "osdmap SORTBITWISE OSDMap flag is NOT set; please set it" @@ -8195,9 +8195,7 @@ void OSD::_committed_osd_maps(epoch_t first, epoch_t last, MOSDMap *m) } } - if ((osdmap->test_flag(CEPH_OSDMAP_NOUP) != - newmap->test_flag(CEPH_OSDMAP_NOUP)) || - (osdmap->is_noup(whoami) != newmap->is_noup(whoami))) { + if (osdmap->is_noup(whoami) != newmap->is_noup(whoami)) { dout(10) << __func__ << " NOUP flag changed in " << newmap->get_epoch() << dendl; if (is_booting()) { diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h index 21051a1cfdaa4..0361771e927ab 100644 --- a/src/osd/OSDMap.h +++ b/src/osd/OSDMap.h @@ -837,64 +837,72 @@ public: unsigned get_crush_node_flags(int id) const; unsigned get_device_class_flags(int id) const; - bool is_noup(int osd) const { + bool is_noup_by_osd(int osd) const { return exists(osd) && (osd_state[osd] & CEPH_OSD_NOUP); } - bool is_nodown(int osd) const { + bool is_nodown_by_osd(int osd) const { return exists(osd) && (osd_state[osd] & CEPH_OSD_NODOWN); } - bool is_noin(int osd) const { + bool is_noin_by_osd(int osd) const { return exists(osd) && (osd_state[osd] & CEPH_OSD_NOIN); } - bool is_noout(int osd) const { + bool is_noout_by_osd(int osd) const { return exists(osd) && (osd_state[osd] & CEPH_OSD_NOOUT); } - void get_noup_osds(vector *osds) const { - ceph_assert(osds); - osds->clear(); - - for (int i = 0; i < max_osd; i++) { - if (is_noup(i)) { - osds->push_back(i); - } - } + bool is_noup(int osd) const { + if (test_flag(CEPH_OSDMAP_NOUP)) // global? + return true; + if (is_noup_by_osd(osd)) // by osd? + return true; + if (get_osd_crush_node_flags(osd) & CEPH_OSD_NOUP) // by crush-node? + return true; + if (auto class_id = crush->get_item_class_id(osd); class_id >= 0 && + get_device_class_flags(class_id) & CEPH_OSD_NOUP) // by device-class? + return true; + return false; } - void get_nodown_osds(vector *osds) const { - ceph_assert(osds); - osds->clear(); - - for (int i = 0; i < max_osd; i++) { - if (is_nodown(i)) { - osds->push_back(i); - } - } + bool is_nodown(int osd) const { + if (test_flag(CEPH_OSDMAP_NODOWN)) + return true; + if (is_nodown_by_osd(osd)) + return true; + if (get_osd_crush_node_flags(osd) & CEPH_OSD_NODOWN) + return true; + if (auto class_id = crush->get_item_class_id(osd); class_id >= 0 && + get_device_class_flags(class_id) & CEPH_OSD_NODOWN) + return true; + return false; } - void get_noin_osds(vector *osds) const { - ceph_assert(osds); - osds->clear(); - - for (int i = 0; i < max_osd; i++) { - if (is_noin(i)) { - osds->push_back(i); - } - } + bool is_noin(int osd) const { + if (test_flag(CEPH_OSDMAP_NOIN)) + return true; + if (is_noin_by_osd(osd)) + return true; + if (get_osd_crush_node_flags(osd) & CEPH_OSD_NOIN) + return true; + if (auto class_id = crush->get_item_class_id(osd); class_id >= 0 && + get_device_class_flags(class_id) & CEPH_OSD_NOIN) + return true; + return false; } - void get_noout_osds(vector *osds) const { - ceph_assert(osds); - osds->clear(); - - for (int i = 0; i < max_osd; i++) { - if (is_noout(i)) { - osds->push_back(i); - } - } + bool is_noout(int osd) const { + if (test_flag(CEPH_OSDMAP_NOOUT)) + return true; + if (is_noout_by_osd(osd)) + return true; + if (get_osd_crush_node_flags(osd) & CEPH_OSD_NOOUT) + return true; + if (auto class_id = crush->get_item_class_id(osd); class_id >= 0 && + get_device_class_flags(class_id) & CEPH_OSD_NOOUT) + return true; + return false; } /** -- 2.39.5