From: xie xingguo Date: Mon, 15 Oct 2018 12:34:56 +0000 (+0800) Subject: crush: get "ceph osd crush class create/rm" back X-Git-Tag: v14.1.0~1171^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=d7ff33e9fd994fced37e487b4eaf8f5cae1cfe58;p=ceph.git crush: get "ceph osd crush class create/rm" back This reverts a27fd9d25cb2819e25cc48b790c40afac0250464 and b863883ca783487401fde4f4480ed1d9b093363e. Quote form Sébastien Han: > IIRC at some point, we were able to create a device class from the CLI. Now it seems that the device class gets created when at least one OSD of a particular class starts. In ceph-ansible, we create pools after the initial monitors are up and we want to assign a device crush class on some of them. That's not possible at the moment since there no device class available yet. Also, someone might want to create its own device class. Something as crazy as running Filestore with a tmpfs osd store and might want to isolate them. I know it's a very limited use case, but still, it could be desired. See also https://www.spinics.net/lists/ceph-devel/msg41152.html Signed-off-by: xie xingguo --- diff --git a/qa/standalone/crush/crush-classes.sh b/qa/standalone/crush/crush-classes.sh index 493649ff4215..2f4ce12e4238 100755 --- a/qa/standalone/crush/crush-classes.sh +++ b/qa/standalone/crush/crush-classes.sh @@ -147,6 +147,17 @@ function TEST_mon_classes() { test "$(get_osds_up rbd SOMETHING)" == "1 2 0" || return 1 add_something $dir SOMETHING || return 1 + # test create and remove class + ceph osd crush class create CLASS || return 1 + ceph osd crush class create CLASS || return 1 # idempotent + ceph osd crush class ls | grep CLASS || return 1 + ceph osd crush class rename CLASS TEMP || return 1 + ceph osd crush class ls | grep TEMP || return 1 + ceph osd crush class rename TEMP CLASS || return 1 + ceph osd crush class ls | grep CLASS || return 1 + ceph osd crush class rm CLASS || return 1 + expect_failure $dir ENOENT ceph osd crush class rm CLASS || return 1 + # test rm-device-class ceph osd crush set-device-class aaa osd.0 || return 1 ceph osd tree | grep -q 'aaa' || return 1 diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc index d4585ce0ea00..0e8e688c87cf 100644 --- a/src/crush/CrushWrapper.cc +++ b/src/crush/CrushWrapper.cc @@ -1536,6 +1536,39 @@ void CrushWrapper::get_subtree_of_type(int type, vector *subtrees) } } +bool CrushWrapper::class_is_in_use(int class_id, ostream *ss) +{ + list rules; + for (unsigned i = 0; i < crush->max_rules; ++i) { + crush_rule *r = crush->rules[i]; + if (!r) + continue; + for (unsigned j = 0; j < r->len; ++j) { + if (r->steps[j].op == CRUSH_RULE_TAKE) { + int root = r->steps[j].arg1; + for (auto &p : class_bucket) { + auto& q = p.second; + if (q.count(class_id) && q[class_id] == root) { + rules.push_back(i); + } + } + } + } + } + if (rules.empty()) { + return false; + } + if (ss) { + ostringstream os; + for (auto &p: rules) { + os << "'" << get_rule_name(p) <<"',"; + } + string out(os.str()); + out.resize(out.size() - 1); // drop last ',' + *ss << "still referenced by crush_rule(s): " << out; + } + return true; +} int CrushWrapper::rename_class(const string& srcname, const string& dstname) { diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h index cbe487ce5376..7e73b361661a 100644 --- a/src/crush/CrushWrapper.h +++ b/src/crush/CrushWrapper.h @@ -1290,6 +1290,7 @@ public: const std::set& used_ids, int *clone, map>> *cmap_item_weight); + bool class_is_in_use(int class_id, ostream *ss = nullptr); int rename_class(const string& srcname, const string& dstname); int populate_classes( const std::map>& old_class_bucket); diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index d63b222555cf..626ab4ed9609 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -551,6 +551,14 @@ COMMAND("osd crush add " \ COMMAND("osd crush set-all-straw-buckets-to-straw2", "convert all CRUSH current straw buckets to use the straw2 algorithm", "osd", "rw", "cli,rest") +COMMAND("osd crush class create " \ + "name=class,type=CephString,goodchars=[A-Za-z0-9-_]", \ + "create crush device class ", \ + "osd", "rw", "cli,rest") +COMMAND("osd crush class rm " \ + "name=class,type=CephString,goodchars=[A-Za-z0-9-_]", \ + "remove crush device class ", \ + "osd", "rw", "cli,rest") COMMAND("osd crush set-device-class " \ "name=class,type=CephString " \ "name=ids,type=CephString,n=N", \ diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index baf1dcaf3820..71da57b44025 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -8552,6 +8552,88 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op, new Monitor::C_Command(mon,op, 0, rs, get_last_committed() + 1)); return true; } + } else if (prefix == "osd crush class create") { + string device_class; + if (!cmd_getval(g_ceph_context, cmdmap, "class", device_class)) { + err = -EINVAL; // no value! + goto reply; + } + if (osdmap.require_osd_release < CEPH_RELEASE_LUMINOUS) { + ss << "you must complete the upgrade and 'ceph osd require-osd-release " + << "luminous' before using crush device classes"; + err = -EPERM; + goto reply; + } + if (!_have_pending_crush() && + _get_stable_crush().class_exists(device_class)) { + ss << "class '" << device_class << "' already exists"; + goto reply; + } + CrushWrapper newcrush; + _get_pending_crush(newcrush); + if (newcrush.class_exists(device_class)) { + ss << "class '" << device_class << "' already exists"; + goto update; + } + int class_id = newcrush.get_or_create_class_id(device_class); + pending_inc.crush.clear(); + newcrush.encode(pending_inc.crush, mon->get_quorum_con_features()); + ss << "created class " << device_class << " with id " << class_id + << " to crush map"; + goto update; + } else if (prefix == "osd crush class rm") { + string device_class; + if (!cmd_getval(g_ceph_context, cmdmap, "class", device_class)) { + err = -EINVAL; // no value! + goto reply; + } + if (osdmap.require_osd_release < CEPH_RELEASE_LUMINOUS) { + ss << "you must complete the upgrade and 'ceph osd require-osd-release " + << "luminous' before using crush device classes"; + err = -EPERM; + goto reply; + } + + CrushWrapper newcrush; + _get_pending_crush(newcrush); + if (!newcrush.class_exists(device_class)) { + err = -ENOENT; + ss << "class '" << device_class << "' does not exist"; + goto reply; + } + int class_id = newcrush.get_class_id(device_class); + stringstream ts; + if (newcrush.class_is_in_use(class_id, &ts)) { + err = -EBUSY; + ss << "class '" << device_class << "' " << ts.str(); + goto reply; + } + + set osds; + newcrush.get_devices_by_class(device_class, &osds); + for (auto& p: osds) { + err = newcrush.remove_device_class(g_ceph_context, p, &ss); + if (err < 0) { + // ss has reason for failure + goto reply; + } + } + + if (osds.empty()) { + // empty class, remove directly + err = newcrush.remove_class_name(device_class); + if (err < 0) { + ss << "class '" << device_class << "' cannot be removed '" + << cpp_strerror(err) << "'"; + goto reply; + } + } + + pending_inc.crush.clear(); + newcrush.encode(pending_inc.crush, mon->get_quorum_con_features()); + ss << "removed class " << device_class << " with id " << class_id + << " from crush map"; + goto update; } else if (prefix == "osd crush class rename") { string srcname, dstname; if (!cmd_getval(cct, cmdmap, "srcname", srcname)) {