From 9d908c14f606239d291cd3551e8f716943e0badf Mon Sep 17 00:00:00 2001 From: xie xingguo Date: Sat, 15 Jul 2017 18:49:25 +0800 Subject: [PATCH] crush: rm-device-class support Signed-off-by: xie xingguo --- qa/standalone/crush/crush-classes.sh | 54 ++++++++++++++++++++++++ src/crush/CrushWrapper.cc | 46 +++++++++++++++++++- src/crush/CrushWrapper.h | 12 +++++- src/mon/MonCommands.h | 5 +++ src/mon/OSDMonitor.cc | 63 ++++++++++++++++++++++++++++ 5 files changed, 177 insertions(+), 3 deletions(-) diff --git a/qa/standalone/crush/crush-classes.sh b/qa/standalone/crush/crush-classes.sh index c2eccc38bd720..0e091fcdac38f 100755 --- a/qa/standalone/crush/crush-classes.sh +++ b/qa/standalone/crush/crush-classes.sh @@ -156,6 +156,43 @@ function TEST_mon_classes() { ceph osd crush class rm CLASS || return 1 expect_failure $dir ENOENT ceph osd crush class rm CLASS || return 1 + # test rm-device-class + ceph osd crush set-device-class aaa osd.0 || return 1 + ceph osd tree | grep -q 'aaa' || return 1 + ceph osd crush dump | grep -q '~aaa' || return 1 + ceph osd crush tree --show-shadow | grep -q '~aaa' || return 1 + ceph osd crush set-device-class bbb osd.1 || return 1 + ceph osd tree | grep -q 'bbb' || return 1 + ceph osd crush dump | grep -q '~bbb' || return 1 + ceph osd crush tree --show-shadow | grep -q '~bbb' || return 1 + ceph osd crush set-device-class ccc osd.2 || return 1 + ceph osd tree | grep -q 'ccc' || return 1 + ceph osd crush dump | grep -q '~ccc' || return 1 + ceph osd crush tree --show-shadow | grep -q '~ccc' || return 1 + ceph osd crush rm-device-class 0 || return 1 + ceph osd tree | grep -q 'aaa' && return 1 + ceph osd crush dump | grep -q '~aaa' && return 1 + ceph osd crush tree --show-shadow | grep -q '~aaa' && return 1 + ceph osd crush class ls | grep -q 'aaa' && return 1 + ceph osd crush rm-device-class 1 || return 1 + ceph osd tree | grep -q 'bbb' && return 1 + ceph osd crush dump | grep -q '~bbb' && return 1 + ceph osd crush tree --show-shadow | grep -q '~bbb' && return 1 + ceph osd crush class ls | grep -q 'bbb' && return 1 + ceph osd crush rm-device-class 2 || return 1 + ceph osd tree | grep -q 'ccc' && return 1 + ceph osd crush dump | grep -q '~ccc' && return 1 + ceph osd crush tree --show-shadow | grep -q '~ccc' && return 1 + ceph osd crush class ls | grep -q 'ccc' && return 1 + ceph osd crush set-device-class asdf all || return 1 + ceph osd tree | grep -q 'asdf' || return 1 + ceph osd crush dump | grep -q '~asdf' || return 1 + ceph osd crush tree --show-shadow | grep -q '~asdf' || return 1 + ceph osd crush rm-device-class all || return 1 + ceph osd tree | grep -q 'asdf' && return 1 + ceph osd crush dump | grep -q '~asdf' && return 1 + ceph osd crush tree --show-shadow | grep -q '~asdf' && return 1 + ceph osd crush set-device-class abc osd.2 || return 1 ceph osd crush move osd.2 root=foo rack=foo-rack host=foo-host || return 1 out=`ceph osd tree |awk '$1 == 2 && $2 == "abc" {print $0}'` @@ -163,6 +200,23 @@ function TEST_mon_classes() { return 1 fi + # verify 'crush move' too + ceph osd crush dump | grep -q 'foo~abc' || return 1 + ceph osd crush tree --show-shadow | grep -q 'foo~abc' || return 1 + ceph osd crush dump | grep -q 'foo-rack~abc' || return 1 + ceph osd crush tree --show-shadow | grep -q 'foo-rack~abc' || return 1 + ceph osd crush dump | grep -q 'foo-host~abc' || return 1 + ceph osd crush tree --show-shadow | grep -q 'foo-host~abc' || return 1 + ceph osd crush rm-device-class osd.2 || return 1 + ceph osd crush dump | grep -q 'foo~abc' && return 1 + ceph osd crush tree --show-shadow | grep -q 'foo~abc' && return 1 + ceph osd crush dump | grep -q 'foo-rack~abc' && return 1 + ceph osd crush tree --show-shadow | grep -q 'foo-rack~abc' && return 1 + ceph osd crush dump | grep -q 'foo-host~abc' && return 1 + ceph osd crush tree --show-shadow | grep -q 'foo-host~abc' && return 1 + # restore class, so we can continue to test create-replicated + ceph osd crush set-device-class abc osd.2 || return 1 + ceph osd crush rule create-replicated foo-rule foo host abc || return 1 # test class_is_in_use diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc index 353dd93d2bea0..8eaf9f6fb8aa8 100644 --- a/src/crush/CrushWrapper.cc +++ b/src/crush/CrushWrapper.cc @@ -1327,14 +1327,19 @@ pair CrushWrapper::get_immediate_parent(int id, int *_ret) return pair(); } -int CrushWrapper::get_immediate_parent_id(int id, int *parent) const +int CrushWrapper::get_immediate_parent_id(int id, + int *parent, + parent_type_t choice) const { for (int bidx = 0; bidx < crush->max_buckets; bidx++) { crush_bucket *b = crush->buckets[bidx]; if (b == 0) continue; - if (is_shadow_item(b->id)) + if (choice == PARENT_NONSHADOW && is_shadow_item(b->id)) { continue; + } else if (choice == PARENT_SHADOW && !is_shadow_item(b->id)) { + continue; + } for (unsigned i = 0; i < b->size; i++) { if (b->items[i] == id) { *parent = b->id; @@ -1812,6 +1817,43 @@ int CrushWrapper::update_device_class(int id, return 1; } +int CrushWrapper::remove_device_class(CephContext *cct, int id, ostream *ss) +{ + assert(ss); + const char *name = get_item_name(id); + if (!name) { + *ss << "osd." << id << " does not have a name"; + return -ENOENT; + } + + const char *class_name = get_item_class(id); + if (!class_name) { + *ss << "osd." << id << " has not been bound to a specific class yet"; + return 0; + } + class_remove_item(id); + + // note that there is no need to remove ourselves from shadow parent + // and reweight because we are going to destroy all shadow trees + // rebuild them all (if necessary) later. + + // see if there is any osds that still reference this class + set devices; + get_devices_by_class(class_name, &devices); + if (devices.empty()) { + // class has no more devices + remove_class_name(class_name); + } + + int r = rebuild_roots_with_classes(); + if (r < 0) { + *ss << "unable to rebuild roots with class '" << class_name << "' " + << "of osd." << id << ": " << cpp_strerror(r); + return r; + } + return 0; +} + int CrushWrapper::device_class_clone(int original_id, int device_class, int *clone) { const char *item_name = get_item_name(original_id); diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h index f69357efa8aa4..8ba61ae428c4e 100644 --- a/src/crush/CrushWrapper.h +++ b/src/crush/CrushWrapper.h @@ -676,7 +676,16 @@ public: * FIXME: ambiguous for items that occur multiple times in the map */ pair get_immediate_parent(int id, int *ret = NULL); - int get_immediate_parent_id(int id, int *parent) const; + + typedef enum { + PARENT_NONSHADOW, + PARENT_SHADOW, + PARENT_ALL, + } parent_type_t; + + int get_immediate_parent_id(int id, + int *parent, + parent_type_t choice = PARENT_NONSHADOW) const; /** * return ancestor of the given type, or 0 if none @@ -1221,6 +1230,7 @@ public: } int update_device_class(int id, const string& class_name, const string& name, ostream *ss); + int remove_device_class(CephContext *cct, int id, ostream *ss); int device_class_clone(int original, int device_class, int *clone); bool class_is_in_use(int class_id, ostream *ss = nullptr); int populate_classes(); diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index e90e1da9b6e7f..b57f9914f992f 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -546,6 +546,11 @@ COMMAND("osd crush set-device-class " \ "set the of the osd(s) [...]," \ "or use to set all.", \ "osd", "rw", "cli,rest") +COMMAND("osd crush rm-device-class " \ + "name=ids,type=CephString,n=N", \ + "remove class of the osd(s) [...]," \ + "or use to remove all.", \ + "osd", "rw", "cli,rest") COMMAND("osd crush create-or-move " \ "name=id,type=CephOsdName " \ "name=weight,type=CephFloat,range=0.0 " \ diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index 8fc05f8619514..2125fa15f2ed7 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -7491,6 +7491,69 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op, return true; } + } else if (prefix == "osd crush rm-device-class") { + bool stop = false; + vector idvec; + cmd_getval(g_ceph_context, cmdmap, "ids", idvec); + CrushWrapper newcrush; + _get_pending_crush(newcrush); + set updated; + + for (unsigned j = 0; j < idvec.size() && !stop; j++) { + set osds; + + // wildcard? + if (j == 0 && + (idvec[0] == "any" || idvec[0] == "all" || idvec[0] == "*")) { + osdmap.get_all_osds(osds); + stop = true; + } else { + // try traditional single osd way + long osd = parse_osd_id(idvec[j].c_str(), &ss); + if (osd < 0) { + // ss has reason for failure + ss << ", unable to parse osd id:\"" << idvec[j] << "\". "; + err = -EINVAL; + goto reply; + } + osds.insert(osd); + } + + for (auto &osd : osds) { + if (!osdmap.exists(osd)) { + ss << "osd." << osd << " does not exist. "; + continue; + } + + auto class_name = newcrush.get_item_class(osd); + stringstream ts; + if (!class_name) { + ss << "osd." << osd << " belongs to no class, "; + continue; + } + // note that we do not verify if class_is_in_use here + // in case the device is misclassified and user wants + // to overridely reset... + + err = newcrush.remove_device_class(g_ceph_context, osd, &ss); + if (err < 0) { + // ss has reason for failure + goto reply; + } + updated.insert(osd); + } + } + + if (!updated.empty()) { + pending_inc.crush.clear(); + newcrush.encode(pending_inc.crush, mon->get_quorum_con_features()); + ss << "done removing class of osd(s): " << updated; + getline(ss, rs); + wait_for_finished_proposal(op, + new Monitor::C_Command(mon,op, 0, rs, get_last_committed() + 1)); + return true; + } + } else if (prefix == "osd crush add-bucket") { // os crush add-bucket string name, typestr; -- 2.39.5