From ee0c8044a6479638cf37c039051a63acd600476f Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 6 Oct 2008 11:07:13 -0700 Subject: [PATCH] dstartnew.sh: use custom crush map with separate metadata osd pool --- src/cm.txt | 105 ++++++++++++++++++++++++++++++++++++++++++++++ src/dstartnew.sh | 7 +++- src/osd/OSDMap.cc | 11 +++-- src/osd/OSDMap.h | 5 ++- src/osdmaptool.cc | 20 +++++---- 5 files changed, 132 insertions(+), 16 deletions(-) create mode 100644 src/cm.txt diff --git a/src/cm.txt b/src/cm.txt new file mode 100644 index 0000000000000..eb8d00646acf0 --- /dev/null +++ b/src/cm.txt @@ -0,0 +1,105 @@ +# begin crush map + +# devices +device 0 device0 offload 1.000 +device 1 device1 offload 1.000 +device 2 device2 offload 1.000 +device 3 device3 offload 1.000 +device 4 device4 offload 1.000 +device 5 device5 offload 1.000 +device 6 device6 offload 1.000 +device 7 device7 offload 1.000 +device 8 device8 offload 1.000 +device 9 device9 offload 1.000 +device 10 device10 offload 1.000 +device 11 device11 offload 1.000 +device 12 device12 offload 1.000 +device 13 device13 offload 1.000 +device 14 device14 offload 1.000 +device 15 device15 offload 1.000 + +# types +type 0 device +type 1 domain +type 2 pool + +# buckets +domain dom0 { + id -1 # do not change unnecessarily + alg uniform # do not change bucket size (4) unnecessarily + item device1 weight 1.000 pos 1 + item device2 weight 1.000 pos 2 + item device3 weight 1.000 pos 3 +} +domain dom1 { + id -2 # do not change unnecessarily + alg uniform # do not change bucket size (4) unnecessarily + item device5 weight 1.000 pos 1 + item device6 weight 1.000 pos 2 + item device7 weight 1.000 pos 3 +} +domain dom2 { + id -3 # do not change unnecessarily + alg uniform # do not change bucket size (4) unnecessarily + item device9 weight 1.000 pos 1 + item device10 weight 1.000 pos 2 + item device11 weight 1.000 pos 3 +} +domain dom3 { + id -4 # do not change unnecessarily + alg uniform # do not change bucket size (4) unnecessarily + item device13 weight 1.000 pos 1 + item device14 weight 1.000 pos 2 + item device15 weight 1.000 pos 3 +} +pool root { + id -5 # do not change unnecessarily + alg list # add new items at the end; do not change order unnecessarily + item dom3 weight 4.000 + item dom2 weight 4.000 + item dom1 weight 4.000 + item dom0 weight 4.000 +} + +pool metaroot { + id -6 + alg list + item device0 weight 1.000 pos 0 + item device4 weight 1.000 pos 0 + item device8 weight 1.000 pos 0 + item device12 weight 1.000 pos 0 +} + + +# rules +rule metadata { + pool 0 + type replicated + min_size 2 + max_size 4 + step take metaroot + step choose firstn 0 type device + step emit +} +rule data { + pool 1 + type replicated + min_size 2 + max_size 4 + step take root + step choose firstn 0 type domain + step choose firstn 1 type device + step emit +} +rule casdata { + pool 2 + type replicated + min_size 2 + max_size 4 + step take root + step choose firstn 0 type domain + step choose firstn 1 type device + step emit +} + +# end crush map diff --git a/src/dstartnew.sh b/src/dstartnew.sh index a3a55c396e45e..5ae4832e520ba 100755 --- a/src/dstartnew.sh +++ b/src/dstartnew.sh @@ -30,7 +30,12 @@ fi ./cmon -d mondata/mon0 --debug_mon 20 --debug_ms 1 # build and inject an initial osd map -./osdmaptool --clobber --createsimple .ceph_monmap 16 .ceph_osdmap # --pgbits 2 +./osdmaptool --clobber --createsimple .ceph_monmap 16 --num_dom 4 .ceph_osdmap + +# use custom crush map to separate data from metadata +./crushtool -c cm.txt -o cm +./osdmaptool --clobber --import-crush cm .ceph_osdmap + ./cmonctl osd setmap -i .ceph_osdmap #ARGS="-m $IP:12345" diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc index 6f8362587c779..a08cd83ffaea5 100644 --- a/src/osd/OSDMap.cc +++ b/src/osd/OSDMap.cc @@ -20,7 +20,7 @@ void OSDMap::build_simple(epoch_t e, ceph_fsid &fsid, - int num_osd, int pg_bits, int lpg_bits, + int num_osd, int num_dom, int pg_bits, int lpg_bits, int mds_local_osd) { dout(10) << "build_simple on " << num_osd @@ -35,7 +35,7 @@ void OSDMap::build_simple(epoch_t e, ceph_fsid &fsid, lpg_num = lpgp_num = lpg_bits ? (1 << (lpg_bits-1)) : 0; // crush map - build_simple_crush_map(crush, num_osd); + build_simple_crush_map(crush, num_osd, num_dom); for (int i=0; i= ndom*3 && num_osd > 8) { int ritems[ndom]; diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h index 5095a08c0d781..f7edd046b869e 100644 --- a/src/osd/OSDMap.h +++ b/src/osd/OSDMap.h @@ -755,9 +755,10 @@ private: * handy helpers to build simple maps... */ void build_simple(epoch_t e, ceph_fsid &fsid, - int num_osd, int pg_bits, int lpg_bits, + int num_osd, int num_dom, + int pg_bits, int lpg_bits, int mds_local_osd); - static void build_simple_crush_map(CrushWrapper& crush, int num_osd); + static void build_simple_crush_map(CrushWrapper& crush, int num_osd, int num_dom=0); }; diff --git a/src/osdmaptool.cc b/src/osdmaptool.cc index 03653c8d43b79..2571127c8facd 100644 --- a/src/osdmaptool.cc +++ b/src/osdmaptool.cc @@ -51,16 +51,16 @@ void printmap(const char *me, OSDMap *m, ostream& out) out << "max_osd " << m->get_max_osd() << "\n"; for (int i=0; iget_max_osd(); i++) { if (m->exists(i)) { - out << "osd" << i - << (m->is_in(i) ? " in":" out") - << (m->is_up(i) ? " up":" down"); - if (m->is_in(i)) - out << " weight " << m->get_weight(i); + out << "osd" << i; + out << (m->is_up(i) ? " up":" down"); if (m->is_up(i)) out << " " << m->get_addr(i); out << " up_from " << m->get_up_from(i) - << " up_thru " << m->get_up_thru(i) - << "\n"; + << " up_thru " << m->get_up_thru(i); + out << (m->is_in(i) ? " in":" out"); + if (m->is_in(i)) + out << " weight " << m->get_weight(i); + out << "\n"; } } out << std::endl; @@ -85,7 +85,7 @@ int main(int argc, const char **argv) bool print = false; bool createsimple = false; const char *monmapfn = 0; - int num_osd = 0; + int num_osd = 0, num_dom = 0; int pg_bits = g_conf.osd_pg_bits; int lpg_bits = g_conf.osd_lpg_bits; bool clobber = false; @@ -107,6 +107,8 @@ int main(int argc, const char **argv) pg_bits = atoi(args[++i]); else if (strcmp(args[i], "--lpg_bits") == 0) lpg_bits = atoi(args[++i]); + else if (strcmp(args[i], "--num_dom") == 0) + num_dom = atoi(args[++i]); else if (strcmp(args[i], "--export-crush") == 0) export_crush = args[++i]; else if (strcmp(args[i], "--import-crush") == 0) @@ -151,7 +153,7 @@ int main(int argc, const char **argv) cerr << me << ": osd count must be > 0" << std::endl; exit(1); } - osdmap.build_simple(0, monmap.fsid, num_osd, pg_bits, lpg_bits, 0); + osdmap.build_simple(0, monmap.fsid, num_osd, num_dom, pg_bits, lpg_bits, 0); modified = true; } -- 2.39.5