From 3a9f43a1a9867c11883cc9a3dcc0f0400189ef02 Mon Sep 17 00:00:00 2001 From: xie xingguo Date: Thu, 25 Apr 2019 17:25:59 +0800 Subject: [PATCH] osd,mon: add 'ceph osd {set,unset}-group ' command Signed-off-by: xie xingguo (cherry picked from commit 9a05a065b936130219dfb7d11282abd80d13de85) --- qa/workunits/cephtool/test.sh | 93 ++++++++++++++++++++++++ src/mon/MonCommands.h | 12 ++++ src/mon/OSDMonitor.cc | 132 ++++++++++++++++++++++++++++++++-- src/osd/OSDMap.cc | 11 ++- src/osd/OSDMap.h | 8 ++- src/test/osd/TestOSDMap.cc | 12 ++-- 6 files changed, 255 insertions(+), 13 deletions(-) diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh index 6751c6bde6ef6..e38c91d77cd5d 100755 --- a/qa/workunits/cephtool/test.sh +++ b/qa/workunits/cephtool/test.sh @@ -1582,6 +1582,99 @@ function test_mon_osd() ceph osd crush rm foo ceph osd dump -f json-pretty | jq ".crush_node_flags" | expect_false grep foo + ceph osd set-group noup osd.0 + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noup' + ceph osd set-group noup,nodown osd.0 + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noup' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'nodown' + ceph osd set-group noup,nodown,noin osd.0 + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noup' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noin' + ceph osd set-group noup,nodown,noin,noout osd.0 + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noup' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noin' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noout' + ceph osd unset-group noup osd.0 + ceph osd dump -f json-pretty | jq ".osds[0].state" | expect_false grep 'noup' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noin' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noout' + ceph osd unset-group noup,nodown osd.0 + ceph osd dump -f json-pretty | jq ".osds[0].state" | expect_false grep 'noup\|nodown' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noin' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noout' + ceph osd unset-group noup,nodown,noin osd.0 + ceph osd dump -f json-pretty | jq ".osds[0].state" | expect_false grep 'noup\|nodown\|noin' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noout' + ceph osd unset-group noup,nodown,noin,noout osd.0 + ceph osd dump -f json-pretty | jq ".osds[0].state" | expect_false grep 'noup\|nodown\|noin\|noout' + + ceph osd set-group noup,nodown,noin,noout osd.0 osd.1 + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noup' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noin' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noout' + ceph osd dump -f json-pretty | jq ".osds[1].state" | grep 'noup' + ceph osd dump -f json-pretty | jq ".osds[1].state" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".osds[1].state" | grep 'noin' + ceph osd dump -f json-pretty | jq ".osds[1].state" | grep 'noout' + ceph osd unset-group noup,nodown,noin,noout osd.0 osd.1 + ceph osd dump -f json-pretty | jq ".osds[0].state" | expect_false grep 'noup\|nodown\|noin\|noout' + ceph osd dump -f json-pretty | jq ".osds[1].state" | expect_false grep 'noup\|nodown\|noin\|noout' + + ceph osd set-group noup all + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noup' + ceph osd unset-group noup all + ceph osd dump -f json-pretty | jq ".osds[0].state" | expect_false grep 'noup' + + # crush node flags + ceph osd crush add-bucket foo host root=default + ceph osd set-group noup foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noup' + ceph osd set-group noup,nodown foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noup' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'nodown' + ceph osd set-group noup,nodown,noin foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noup' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noin' + ceph osd set-group noup,nodown,noin,noout foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noup' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noin' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noout' + + ceph osd unset-group noup foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | expect_false grep 'noup' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noin' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noout' + ceph osd unset-group noup,nodown foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | expect_false grep 'noup\|nodown' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noin' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noout' + ceph osd unset-group noup,nodown,noin foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | expect_false grep 'noup\|nodown\|noin' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noout' + ceph osd unset-group noup,nodown,noin,noout foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | expect_false grep 'noup\|nodown\|noin\|noout' + + ceph osd set-group noin,noout foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noin' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noout' + ceph osd unset-group noin,noout foo + ceph osd dump -f json-pretty | jq ".crush_node_flags" | expect_false grep 'foo' + + ceph osd set-group noup,nodown,noin,noout foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noup' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noin' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noout' + ceph osd crush rm foo + ceph osd dump -f json-pretty | jq ".crush_node_flags" | expect_false grep 'foo' + # make sure mark out preserves weight ceph osd reweight osd.0 .5 ceph osd dump | grep ^osd.0 | grep 'weight 0.5' diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index af0352cecf4bd..d7eeea64a3715 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -872,6 +872,18 @@ COMMAND("osd rm-noout " \ "(if they are currently marked as noout), " \ "can use to automatically filter out all noout osds", \ "osd", "rw") +COMMAND("osd set-group " \ + "name=flags,type=CephString " + "name=who,type=CephString,n=N", \ + "set for batch osds or crush nodes, " \ + " must be a comma-separated subset of {noup,nodown,noin,noout}", \ + "osd", "rw") +COMMAND("osd unset-group " \ + "name=flags,type=CephString " + "name=who,type=CephString,n=N", \ + "unset for batch osds or crush nodes, " \ + " must be a comma-separated subset of {noup,nodown,noin,noout}", \ + "osd", "rw") COMMAND("osd reweight " \ "name=id,type=CephOsdName " \ "type=CephFloat,name=weight,range=0.0|1.0", \ diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index 8f5fa2170519b..db0ff48f21a20 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -2453,7 +2453,7 @@ bool OSDMonitor::can_mark_down(int i) return false; } - if (osdmap.get_crush_node_flags(i) & CEPH_OSD_NODOWN) { + if (osdmap.get_osd_crush_node_flags(i) & CEPH_OSD_NODOWN) { dout(5) << __func__ << " osd." << i << " is marked as nodown via a crush node flag, " << "will not mark it down" << dendl; @@ -2490,7 +2490,7 @@ bool OSDMonitor::can_mark_up(int i) return false; } - if (osdmap.get_crush_node_flags(i) & CEPH_OSD_NOUP) { + if (osdmap.get_osd_crush_node_flags(i) & CEPH_OSD_NOUP) { dout(5) << __func__ << " osd." << i << " is marked as noup via a crush node flag, " << "will not mark it up" << dendl; @@ -2517,7 +2517,7 @@ bool OSDMonitor::can_mark_out(int i) return false; } - if (osdmap.get_crush_node_flags(i) & CEPH_OSD_NOOUT) { + if (osdmap.get_osd_crush_node_flags(i) & CEPH_OSD_NOOUT) { dout(5) << __func__ << " osd." << i << " is marked as noout via a crush node flag, " << "will not mark it out" << dendl; @@ -2560,7 +2560,7 @@ bool OSDMonitor::can_mark_in(int i) return false; } - if (osdmap.get_crush_node_flags(i) & CEPH_OSD_NOIN) { + if (osdmap.get_osd_crush_node_flags(i) & CEPH_OSD_NOIN) { dout(5) << __func__ << " osd." << i << " is marked as noin via a crush node flag, " << "will not mark it in" << dendl; @@ -10532,6 +10532,130 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op, get_last_committed() + 1)); return true; } + } else if (prefix == "osd set-group" || + prefix == "osd unset-group") { + bool do_set = prefix == "osd set-group"; + string flag_str; + vector who; + cmd_getval(cct, cmdmap, "flags", flag_str); + cmd_getval(cct, cmdmap, "who", who); + vector raw_flags; + boost::split(raw_flags, flag_str, boost::is_any_of(",")); + unsigned flags = 0; + for (auto& f : raw_flags) { + if (f == "noup") + flags |= CEPH_OSD_NOUP; + else if (f == "nodown") + flags |= CEPH_OSD_NODOWN; + else if (f == "noin") + flags |= CEPH_OSD_NOIN; + else if (f == "noout") + flags |= CEPH_OSD_NOOUT; + else { + ss << "unrecognized flag '" << f << "', must be one of " + << "{noup,nodown,noin,noout}"; + err = -EINVAL; + goto reply; + } + } + if (flags == 0) { + ss << "must specify flag(s) {noup,nodwon,noin,noout} to set/unset"; + err = -EINVAL; + goto reply; + } + if (who.empty()) { + ss << "must specify at least one or more targets to set/unset"; + err = -EINVAL; + goto reply; + } + set osds; + set crush_nodes; + for (auto& w : who) { + if (w == "any" || w == "all" || w == "*") { + osdmap.get_all_osds(osds); + break; + } + std::stringstream ts; + if (auto osd = parse_osd_id(w.c_str(), &ts); osd >= 0) { + osds.insert(osd); + } else if (osdmap.crush->name_exists(w)) { + crush_nodes.insert(osdmap.crush->get_item_id(w)); + } else { + ss << "unable to parse osd id or crush node:\"" << w << "\". "; + } + } + if (osds.empty() && crush_nodes.empty()) { + // ss has reason for failure + err = -EINVAL; + goto reply; + } + bool any = false; + for (auto osd : osds) { + if (!osdmap.exists(osd)) { + ss << "osd." << osd << " does not exist. "; + continue; + } + if (do_set) { + if (flags & CEPH_OSD_NOUP) { + any |= osdmap.is_noup(osd) ? + pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOUP) : + pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOUP); + } + if (flags & CEPH_OSD_NODOWN) { + any |= osdmap.is_nodown(osd) ? + pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NODOWN) : + pending_inc.pending_osd_state_set(osd, CEPH_OSD_NODOWN); + } + if (flags & CEPH_OSD_NOIN) { + any |= osdmap.is_noin(osd) ? + pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOIN) : + pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOIN); + } + if (flags & CEPH_OSD_NOOUT) { + any |= osdmap.is_noout(osd) ? + pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOOUT) : + pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOOUT); + } + } else { + if (flags & CEPH_OSD_NOUP) { + any |= osdmap.is_noup(osd) ? + pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOUP) : + pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOUP); + } + if (flags & CEPH_OSD_NODOWN) { + any |= osdmap.is_nodown(osd) ? + pending_inc.pending_osd_state_set(osd, CEPH_OSD_NODOWN) : + pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NODOWN); + } + if (flags & CEPH_OSD_NOIN) { + any |= osdmap.is_noin(osd) ? + pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOIN) : + pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOIN); + } + if (flags & CEPH_OSD_NOOUT) { + any |= osdmap.is_noout(osd) ? + pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOOUT) : + pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOOUT); + } + } + } + for (auto& id : crush_nodes) { + auto old_flags = osdmap.get_crush_node_flags(id); + auto& pending_flags = pending_inc.new_crush_node_flags[id]; + pending_flags |= old_flags; // adopt existing flags first! + if (do_set) { + pending_flags |= flags; + } else { + pending_flags &= ~flags; + } + any = true; + } + if (any) { + getline(ss, rs); + wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, err, rs, + get_last_committed() + 1)); + return true; + } } else if (prefix == "osd add-noup" || prefix == "osd add-nodown" || prefix == "osd add-noin" || diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc index 62a3ef32e3d68..d846231164ff3 100644 --- a/src/osd/OSDMap.cc +++ b/src/osd/OSDMap.cc @@ -5826,7 +5826,7 @@ float OSDMap::pool_raw_used_rate(int64_t poolid) const } } -unsigned OSDMap::get_crush_node_flags(int osd) const +unsigned OSDMap::get_osd_crush_node_flags(int osd) const { unsigned flags = 0; if (!crush_node_flags.empty()) { @@ -5842,3 +5842,12 @@ unsigned OSDMap::get_crush_node_flags(int osd) const } return flags; } + +unsigned OSDMap::get_crush_node_flags(int id) const +{ + unsigned flags = 0; + auto it = crush_node_flags.find(id); + if (it != crush_node_flags.end()) + flags = it->second; + return flags; +} diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h index 9d7d75bec30d9..1cc3a1479aa11 100644 --- a/src/osd/OSDMap.h +++ b/src/osd/OSDMap.h @@ -476,8 +476,11 @@ public: return new_state.count(osd) && (new_state[osd] & state) != 0; } - void pending_osd_state_set(int osd, unsigned state) { + bool pending_osd_state_set(int osd, unsigned state) { + if (pending_osd_has_state(osd, state)) + return false; new_state[osd] |= state; + return true; } // cancel the specified pending osd state if there is any @@ -828,7 +831,8 @@ public: return !is_out(osd); } - unsigned get_crush_node_flags(int osd) const; + unsigned get_osd_crush_node_flags(int osd) const; + unsigned get_crush_node_flags(int id) const; bool is_noup(int osd) const { return exists(osd) && (osd_state[osd] & CEPH_OSD_NOUP); diff --git a/src/test/osd/TestOSDMap.cc b/src/test/osd/TestOSDMap.cc index 0967a536a24c6..0ed5b477ccf49 100644 --- a/src/test/osd/TestOSDMap.cc +++ b/src/test/osd/TestOSDMap.cc @@ -509,28 +509,28 @@ TEST_F(OSDMapTest, PrimaryAffinity) { } } -TEST_F(OSDMapTest, get_crush_node_flags) { +TEST_F(OSDMapTest, get_osd_crush_node_flags) { set_up_map(); for (unsigned i=0; i