OSDMap: respect default replicated ruleset config opt in build_simple()

Use id provided by osd_pool_default_crush_{replicated_ruleset,rule}
config options when creating a simple replicated ruleset for an initial
osdmap instead of always making it ruleset 0.  Not doing so may leave
default created pools (currently "rbd") in a broken state with their
crush_ruleset pointing to a non-existent ruleset.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>

diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc
index 497c2196958a2f2404b8d9b1c511885da8b177f8..fcf8b59c7d630582f5d4c1605f8790ba0797c689 100644 (file)
--- a/src/crush/CrushWrapper.cc
+++ b/src/crush/CrushWrapper.cc
@@ -900,17 +900,33 @@ void CrushWrapper::reweight(CephContext *cct)
   }
 }
 
-int CrushWrapper::add_simple_ruleset(string name, string root_name,
-                                     string failure_domain_name,
-                                     string mode,
-                                     int rule_type,
-                                     ostream *err)
+int CrushWrapper::add_simple_ruleset_at(string name, string root_name,
+                                        string failure_domain_name,
+                                        string mode, int rule_type,
+                                        int rno, ostream *err)
 {
   if (rule_exists(name)) {
     if (err)
       *err << "rule " << name << " exists";
     return -EEXIST;
   }
+  if (rno >= 0) {
+    if (rule_exists(rno)) {
+      if (err)
+        *err << "rule with ruleno " << rno << " exists";
+      return -EEXIST;
+    }
+    if (ruleset_exists(rno)) {
+      if (err)
+        *err << "ruleset " << rno << " exists";
+      return -EEXIST;
+    }
+  } else {
+    for (rno = 0; rno < get_max_rules(); rno++) {
+      if (!rule_exists(rno) && !ruleset_exists(rno))
+        break;
+    }
+  }
   if (!name_exists(root_name)) {
     if (err)
       *err << "root item " << root_name << " does not exist";
@@ -932,11 +948,6 @@ int CrushWrapper::add_simple_ruleset(string name, string root_name,
     return -EINVAL;
   }
 
-  int rno = -1;
-  for (rno = 0; rno < get_max_rules(); rno++) {
-    if (!rule_exists(rno) && !ruleset_exists(rno))
-       break;
-  }
   int steps = 3;
   if (mode == "indep")
     steps = 5;
@@ -975,6 +986,15 @@ int CrushWrapper::add_simple_ruleset(string name, string root_name,
   return rno;
 }
 
+int CrushWrapper::add_simple_ruleset(string name, string root_name,
+                                     string failure_domain_name,
+                                     string mode, int rule_type,
+                                     ostream *err)
+{
+  return add_simple_ruleset_at(name, root_name, failure_domain_name, mode,
+                               rule_type, -1, err);
+}
+
 int CrushWrapper::get_rule_weight_osd_map(unsigned ruleno, map<int,float> *pmap)
 {
   if (ruleno >= crush->max_rules)
@@ -1593,16 +1613,13 @@ void CrushWrapper::generate_test_instances(list<CrushWrapper*>& o)
   // fixme
 }
 
-/**
- * Determine the default CRUSH ruleset ID to be used with
- * newly created replicated pools.
- *
- * @returns a ruleset ID (>=0) or an error (<0)
- */
-int CrushWrapper::get_osd_pool_default_crush_replicated_ruleset(CephContext *cct)
+int CrushWrapper::_get_osd_pool_default_crush_replicated_ruleset(CephContext *cct,
+                                                                 bool quiet)
 {
-  int crush_ruleset = cct->_conf->osd_pool_default_crush_replicated_ruleset;
-  if (cct->_conf->osd_pool_default_crush_rule != -1) {
+  int crush_ruleset = cct->_conf->osd_pool_default_crush_rule;
+  if (crush_ruleset == -1) {
+    crush_ruleset = cct->_conf->osd_pool_default_crush_replicated_ruleset;
+  } else if (!quiet) {
     ldout(cct, 0) << "osd_pool_default_crush_rule is deprecated "
                   << "use osd_pool_default_crush_replicated_ruleset instead"
                   << dendl;
@@ -1611,9 +1628,21 @@ int CrushWrapper::get_osd_pool_default_crush_replicated_ruleset(CephContext *cct
                   << "osd_pool_default_crush_replicated_ruleset = "
                   << cct->_conf->osd_pool_default_crush_replicated_ruleset
                   << dendl;
-    crush_ruleset = cct->_conf->osd_pool_default_crush_rule;
   }
 
+  return crush_ruleset;
+}
+
+/**
+ * Determine the default CRUSH ruleset ID to be used with
+ * newly created replicated pools.
+ *
+ * @returns a ruleset ID (>=0) or -1 if no suitable ruleset found
+ */
+int CrushWrapper::get_osd_pool_default_crush_replicated_ruleset(CephContext *cct)
+{
+  int crush_ruleset = _get_osd_pool_default_crush_replicated_ruleset(cct,
+                                                                     false);
   if (crush_ruleset == CEPH_DEFAULT_CRUSH_REPLICATED_RULESET) {
     crush_ruleset = find_first_ruleset(pg_pool_t::TYPE_REPLICATED);
   }

diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h
index 4e60d7ec42ca40859b69a4119fa7b21095f3eab7..2169a4c52798e39ed9f86bf9225ab063408edfb5 100644 (file)
--- a/src/crush/CrushWrapper.h
+++ b/src/crush/CrushWrapper.h
@@ -796,6 +796,12 @@ public:
 
   int add_simple_ruleset(string name, string root_name, string failure_domain_type,
                         string mode, int rule_type, ostream *err = 0);
+  /**
+   * @param rno ruleset id to use, -1 to pick the lowest available
+   */
+  int add_simple_ruleset_at(string name, string root_name,
+                            string failure_domain_type, string mode,
+                            int rule_type, int rno, ostream *err = 0);
 
   int remove_rule(int ruleno);
 
@@ -1055,6 +1061,8 @@ public:
   void dump_tree(ostream *out, Formatter *f) const;
   static void generate_test_instances(list<CrushWrapper*>& o);
 
+  int _get_osd_pool_default_crush_replicated_ruleset(CephContext *cct,
+                                                     bool quiet);
   int get_osd_pool_default_crush_replicated_ruleset(CephContext *cct);
 
   static bool is_valid_crush_name(const string& s);

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index cab30e36017279c37077a42f1f9a1d508078f01d..963f6ea6733e433a01be6c54baa7913a0472e04c 100644 (file)
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -2857,12 +2857,18 @@ int OSDMap::build_simple_crush_rulesets(CephContext *cct,
                                        const string& root,
                                        ostream *ss)
 {
+  int crush_ruleset =
+      crush._get_osd_pool_default_crush_replicated_ruleset(cct, true);
   string failure_domain =
     crush.get_type_name(cct->_conf->osd_crush_chooseleaf_type);
 
+  if (crush_ruleset == CEPH_DEFAULT_CRUSH_REPLICATED_RULESET)
+    crush_ruleset = -1; // create ruleset 0 by default
+
   int r;
-  r = crush.add_simple_ruleset("replicated_ruleset", root, failure_domain,
-                              "firstn", pg_pool_t::TYPE_REPLICATED, ss);
+  r = crush.add_simple_ruleset_at("replicated_ruleset", root, failure_domain,
+                                  "firstn", pg_pool_t::TYPE_REPLICATED,
+                                  crush_ruleset, ss);
   if (r < 0)
     return r;
   // do not add an erasure rule by default or else we will implicitly

diff --git a/src/test/mon/osd-pool-create.sh b/src/test/mon/osd-pool-create.sh
index 59e534e5f59db31010773da218e1d3c21f6f3839..820d00c20d94bd5b88f516ca93fa8844e6a0ca4c 100755 (executable)
--- a/src/test/mon/osd-pool-create.sh
+++ b/src/test/mon/osd-pool-create.sh
@@ -41,6 +41,7 @@ function TEST_default_deprectated_0() {
     run_mon $dir a \
         --osd_pool_default_crush_replicated_ruleset $expected || return 1
     ./ceph osd pool get rbd crush_ruleset | grep 'ruleset: '$expected || return 1
+    ./ceph osd crush rule dump replicated_ruleset | grep '"ruleset": '$expected || return 1
     CEPH_ARGS='' ./ceph --admin-daemon $dir/ceph-mon.a.asok log flush || return 1
     ! grep "osd_pool_default_crush_rule is deprecated " $dir/mon.a.log || return 1
 }
@@ -52,6 +53,7 @@ function TEST_default_deprectated_1() {
     run_mon $dir a \
         --osd_pool_default_crush_rule $expected || return 1
     ./ceph osd pool get rbd crush_ruleset | grep 'ruleset: '$expected || return 1
+    ./ceph osd crush rule dump replicated_ruleset | grep '"ruleset": '$expected || return 1
     CEPH_ARGS='' ./ceph --admin-daemon $dir/ceph-mon.a.asok log flush || return 1
     grep "osd_pool_default_crush_rule is deprecated " $dir/mon.a.log || return 1
 }
@@ -65,6 +67,7 @@ function TEST_default_deprectated_2() {
         --osd_pool_default_crush_replicated_ruleset $unexpected || return 1
     ./ceph osd pool get rbd crush_ruleset | grep 'ruleset: '$expected || return 1
     ! ./ceph --format json osd dump | grep '"crush_ruleset":'$unexpected || return 1
+    ./ceph osd crush rule dump replicated_ruleset | grep '"ruleset": '$expected || return 1
     CEPH_ARGS='' ./ceph --admin-daemon $dir/ceph-mon.a.asok log flush || return 1
     grep "osd_pool_default_crush_rule is deprecated " $dir/mon.a.log || return 1
 }