osdmap: make replica separate in default crush map configurable

Add 'osd crush chooseleaf type' option to control what the default
CRUSH rule separates replicas across.  Default to 1 (host), and set it
to 0 in vstart.sh.

Fixes: #3785
Signed-off-by: Sage Weil <sage@inktank.com>
Reviewed-by: Greg Farnum <greg@inktank.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 1edd4f57f6eae37964714da13f7c3817744b36f3..47c1ff1d150a4540890b59251c1457fabf0fb878 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -301,6 +301,7 @@ OPTION(osd_max_pgls, OPT_U64, 1024) // max number of pgls entries to return
 OPTION(osd_client_message_size_cap, OPT_U64, 500*1024L*1024L) // client data allowed in-memory (in bytes)
 OPTION(osd_pg_bits, OPT_INT, 6)  // bits per osd
 OPTION(osd_pgp_bits, OPT_INT, 6)  // bits per osd
+OPTION(osd_crush_chooseleaf_type, OPT_INT, 1) // 1 = host
 OPTION(osd_min_rep, OPT_INT, 1)
 OPTION(osd_max_rep, OPT_INT, 10)
 OPTION(osd_pool_default_crush_rule, OPT_INT, 0)

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 63a1fae809bc45524fa9512aa68e488f5970cef4..439ff06505ae0cf99fcece6842ef87ad61228602 100644 (file)
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -1680,8 +1680,10 @@ void OSDMap::build_simple_crush_map(CephContext *cct, CrushWrapper& crush,
     crush_rule *rule = crush_make_rule(3, ruleset, pg_pool_t::TYPE_REP, minrep, maxrep);
     assert(rule);
     crush_rule_set_step(rule, 0, CRUSH_RULE_TAKE, rootid, 0);
-    // just spread across osds
-    crush_rule_set_step(rule, 1, CRUSH_RULE_CHOOSE_FIRSTN, CRUSH_CHOOSE_N, 0);
+    crush_rule_set_step(rule, 1,
+                       cct->_conf->osd_crush_chooseleaf_type ? CRUSH_RULE_CHOOSE_LEAF_FIRSTN : CRUSH_RULE_CHOOSE_FIRSTN,
+                       CRUSH_CHOOSE_N,
+                       cct->_conf->osd_crush_chooseleaf_type);
     crush_rule_set_step(rule, 2, CRUSH_RULE_EMIT, 0, 0);
     int rno = crush_add_rule(crush.crush, rule, -1);
     crush.set_rule_name(rno, p->second);

diff --git a/src/vstart.sh b/src/vstart.sh
index e0516341e1013a7536572a27ffcba539d21a3783..79095b5218d29a8c6d6da9a0a88e1acfd423f1d3 100755 (executable)
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -290,6 +290,7 @@ if [ "$start_mon" -eq 1 ]; then
 [global]
         osd pg bits = 3
         osd pgp bits = 5  ; (invalid, but ceph should cope!)
+        osd crush chooseleaf type = 0
         osd pool default min size = 1
 EOF
 if [ "$cephx" -eq 1 ] ; then