dstartnew.sh: use custom crush map with separate metadata osd pool

diff --git a/src/cm.txt b/src/cm.txt
new file mode 100644 (file)
index 0000000..eb8d006
--- /dev/null
+++ b/src/cm.txt
@@ -0,0 +1,105 @@
+# begin crush map
+
+# devices
+device 0 device0 offload 1.000
+device 1 device1 offload 1.000
+device 2 device2 offload 1.000
+device 3 device3 offload 1.000
+device 4 device4 offload 1.000
+device 5 device5 offload 1.000
+device 6 device6 offload 1.000
+device 7 device7 offload 1.000
+device 8 device8 offload 1.000
+device 9 device9 offload 1.000
+device 10 device10 offload 1.000
+device 11 device11 offload 1.000
+device 12 device12 offload 1.000
+device 13 device13 offload 1.000
+device 14 device14 offload 1.000
+device 15 device15 offload 1.000
+
+# types
+type 0 device
+type 1 domain
+type 2 pool
+
+# buckets
+domain dom0 {
+       id -1           # do not change unnecessarily
+       alg uniform     # do not change bucket size (4) unnecessarily
+       item device1 weight 1.000 pos 1
+       item device2 weight 1.000 pos 2
+       item device3 weight 1.000 pos 3
+}
+domain dom1 {
+       id -2           # do not change unnecessarily
+       alg uniform     # do not change bucket size (4) unnecessarily
+       item device5 weight 1.000 pos 1
+       item device6 weight 1.000 pos 2
+       item device7 weight 1.000 pos 3
+}
+domain dom2 {
+       id -3           # do not change unnecessarily
+       alg uniform     # do not change bucket size (4) unnecessarily
+       item device9 weight 1.000 pos 1
+       item device10 weight 1.000 pos 2
+       item device11 weight 1.000 pos 3
+}
+domain dom3 {
+       id -4           # do not change unnecessarily
+       alg uniform     # do not change bucket size (4) unnecessarily
+       item device13 weight 1.000 pos 1
+       item device14 weight 1.000 pos 2
+       item device15 weight 1.000 pos 3
+}
+pool root {
+       id -5           # do not change unnecessarily
+       alg list        # add new items at the end; do not change order unnecessarily
+       item dom3 weight 4.000
+       item dom2 weight 4.000
+       item dom1 weight 4.000
+       item dom0 weight 4.000
+}
+
+pool metaroot {
+       id -6
+       alg list
+       item device0 weight 1.000 pos 0
+       item device4 weight 1.000 pos 0
+       item device8 weight 1.000 pos 0
+       item device12 weight 1.000 pos 0
+}
+
+
+# rules
+rule metadata {
+       pool 0
+       type replicated
+       min_size 2
+       max_size 4
+       step take metaroot
+       step choose firstn 0 type device
+       step emit
+}
+rule data {
+       pool 1
+       type replicated
+       min_size 2
+       max_size 4
+       step take root
+       step choose firstn 0 type domain
+       step choose firstn 1 type device
+       step emit
+}
+rule casdata {
+       pool 2
+       type replicated
+       min_size 2
+       max_size 4
+       step take root
+       step choose firstn 0 type domain
+       step choose firstn 1 type device
+       step emit
+}
+
+# end crush map

diff --git a/src/dstartnew.sh b/src/dstartnew.sh
index a3a55c396e45e0c9399d2bddd25ba0978949cc0b..5ae4832e520ba9aa61f4c5d0bcd19e841dbbe9e0 100755 (executable)
--- a/src/dstartnew.sh
+++ b/src/dstartnew.sh
@@ -30,7 +30,12 @@ fi
 ./cmon -d mondata/mon0 --debug_mon 20 --debug_ms 1
 
 # build and inject an initial osd map
-./osdmaptool --clobber --createsimple .ceph_monmap 16 .ceph_osdmap # --pgbits 2
+./osdmaptool --clobber --createsimple .ceph_monmap 16 --num_dom 4 .ceph_osdmap
+
+# use custom crush map to separate data from metadata
+./crushtool -c cm.txt -o cm
+./osdmaptool --clobber --import-crush cm .ceph_osdmap
+
 ./cmonctl osd setmap -i .ceph_osdmap
 
 #ARGS="-m $IP:12345"

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 6f8362587c779f121f59dee828895c0a21db0a5e..a08cd83ffaea550f3dce70599a5157cb8dcd9038 100644 (file)
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -20,7 +20,7 @@
 
 
 void OSDMap::build_simple(epoch_t e, ceph_fsid &fsid,
-                         int num_osd, int pg_bits, int lpg_bits,
+                         int num_osd, int num_dom, int pg_bits, int lpg_bits,
                          int mds_local_osd)
 {
   dout(10) << "build_simple on " << num_osd
@@ -35,7 +35,7 @@ void OSDMap::build_simple(epoch_t e, ceph_fsid &fsid,
   lpg_num = lpgp_num = lpg_bits ? (1 << (lpg_bits-1)) : 0;
   
   // crush map
-  build_simple_crush_map(crush, num_osd);
+  build_simple_crush_map(crush, num_osd, num_dom);
 
   for (int i=0; i<num_osd; i++) {
     set_state(i, CEPH_OSD_EXISTS|CEPH_OSD_CLEAN);
@@ -53,7 +53,8 @@ void OSDMap::build_simple(epoch_t e, ceph_fsid &fsid,
   }
 }
 
-void OSDMap::build_simple_crush_map(CrushWrapper& crush, int num_osd)
+void OSDMap::build_simple_crush_map(CrushWrapper& crush, int num_osd,
+                                   int num_dom)
 {
   // new
   crush.create();
@@ -64,7 +65,9 @@ void OSDMap::build_simple_crush_map(CrushWrapper& crush, int num_osd)
   int npools = 3;
 
   int minrep = g_conf.osd_min_rep;
-  int ndom = MAX(g_conf.osd_max_rep, g_conf.osd_max_raid_width);
+  int ndom = num_dom;
+  if (!ndom)
+    MAX(g_conf.osd_max_rep, g_conf.osd_max_raid_width);
   if (num_osd >= ndom*3 &&
       num_osd > 8) {
     int ritems[ndom];

diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index 5095a08c0d7810160ed090ff523825143302f955..f7edd046b869e66a5b5ddf24a5c24aeac97c65f2 100644 (file)
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -755,9 +755,10 @@ private:
    * handy helpers to build simple maps...
    */
   void build_simple(epoch_t e, ceph_fsid &fsid,
-                   int num_osd, int pg_bits, int lpg_bits,
+                   int num_osd, int num_dom,
+                   int pg_bits, int lpg_bits,
                    int mds_local_osd);
-  static void build_simple_crush_map(CrushWrapper& crush, int num_osd);
+  static void build_simple_crush_map(CrushWrapper& crush, int num_osd, int num_dom=0);
 
 };
 

diff --git a/src/osdmaptool.cc b/src/osdmaptool.cc
index 03653c8d43b792bb6bc499d903e9f49dae5dcae1..2571127c8facd84dc0fdd132e474850534e3460c 100644 (file)
--- a/src/osdmaptool.cc
+++ b/src/osdmaptool.cc
@@ -51,16 +51,16 @@ void printmap(const char *me, OSDMap *m, ostream& out)
   out << "max_osd " << m->get_max_osd() << "\n";
   for (int i=0; i<m->get_max_osd(); i++) {
     if (m->exists(i)) {
-      out << "osd" << i
-         << (m->is_in(i) ? " in":" out")
-         << (m->is_up(i) ? " up":" down");
-      if (m->is_in(i))
-       out << " weight " << m->get_weight(i);
+      out << "osd" << i;
+      out << (m->is_up(i) ? " up":" down");
       if (m->is_up(i))
        out << " " << m->get_addr(i);
       out << " up_from " << m->get_up_from(i)
-         << " up_thru " << m->get_up_thru(i)
-         << "\n";
+         << " up_thru " << m->get_up_thru(i);
+      out << (m->is_in(i) ? " in":" out");
+      if (m->is_in(i))
+       out << " weight " << m->get_weight(i);
+      out << "\n";
     }
   }
   out << std::endl;
@@ -85,7 +85,7 @@ int main(int argc, const char **argv)
   bool print = false;
   bool createsimple = false;
   const char *monmapfn = 0;
-  int num_osd = 0;
+  int num_osd = 0, num_dom = 0;
   int pg_bits = g_conf.osd_pg_bits;
   int lpg_bits = g_conf.osd_lpg_bits;
   bool clobber = false;
@@ -107,6 +107,8 @@ int main(int argc, const char **argv)
       pg_bits = atoi(args[++i]);
     else if (strcmp(args[i], "--lpg_bits") == 0)
       lpg_bits = atoi(args[++i]);
+    else if (strcmp(args[i], "--num_dom") == 0)
+      num_dom = atoi(args[++i]);
     else if (strcmp(args[i], "--export-crush") == 0)
       export_crush = args[++i];
     else if (strcmp(args[i], "--import-crush") == 0)
@@ -151,7 +153,7 @@ int main(int argc, const char **argv)
       cerr << me << ": osd count must be > 0" << std::endl;
       exit(1);
     }
-    osdmap.build_simple(0, monmap.fsid, num_osd, pg_bits, lpg_bits, 0);
+    osdmap.build_simple(0, monmap.fsid, num_osd, num_dom, pg_bits, lpg_bits, 0);
     modified = true;
   }