From: Sage Weil Date: Sun, 19 Feb 2012 22:16:23 +0000 (-0800) Subject: crush: move (de)compile into CrushCompiler class X-Git-Tag: v0.43~46^2~5 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e42a0e9f59f0a0ddf19c7f0d462cb71246ac0bc6;p=ceph.git crush: move (de)compile into CrushCompiler class Signed-off-by: Sage Weil --- diff --git a/src/Makefile.am b/src/Makefile.am index 2198605691636..085507e22aee8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -859,7 +859,8 @@ crush_files = \ crush/mapper.c \ crush/crush.c \ crush/hash.c \ - crush/CrushWrapper.cc + crush/CrushWrapper.cc \ + crush/CrushCompiler.cc # this list ommits the ceph_ver.c file libcommon_files = \ @@ -1144,6 +1145,7 @@ noinst_HEADERS = \ common/secret.h\ common/strtol.h\ common/static_assert.h\ + crush/CrushCompiler.h\ crush/CrushWrapper.h\ crush/CrushWrapper.i\ crush/builder.h\ diff --git a/src/crush/CrushCompiler.cc b/src/crush/CrushCompiler.cc new file mode 100644 index 0000000000000..cc2275c4adc3c --- /dev/null +++ b/src/crush/CrushCompiler.cc @@ -0,0 +1,725 @@ + +#include "CrushCompiler.h" + +#ifndef EBADE +#define EBADE EFTYPE +#endif + +#include +#include +#include +#include +#include +#include + +#include + +// ------------- + +static void print_type_name(ostream& out, int t, CrushWrapper &crush) +{ + const char *name = crush.get_type_name(t); + if (name) + out << name; + else if (t == 0) + out << "device"; + else + out << "type" << t; +} + +static void print_item_name(ostream& out, int t, CrushWrapper &crush) +{ + const char *name = crush.get_item_name(t); + if (name) + out << name; + else if (t >= 0) + out << "device" << t; + else + out << "bucket" << (-1-t); +} + +static void print_rule_name(ostream& out, int t, CrushWrapper &crush) +{ + const char *name = crush.get_rule_name(t); + if (name) + out << name; + else + out << "rule" << t; +} + +static void print_fixedpoint(ostream& out, int i) +{ + char s[20]; + snprintf(s, sizeof(s), "%.3f", (float)i / (float)0x10000); + out << s; +} + +int CrushCompiler::decompile_bucket_impl(int i, ostream &out) +{ + int type = crush.get_bucket_type(i); + print_type_name(out, type, crush); + out << " "; + print_item_name(out, i, crush); + out << " {\n"; + out << "\tid " << i << "\t\t# do not change unnecessarily\n"; + + out << "\t# weight "; + print_fixedpoint(out, crush.get_bucket_weight(i)); + out << "\n"; + + int n = crush.get_bucket_size(i); + + int alg = crush.get_bucket_alg(i); + out << "\talg " << crush_bucket_alg_name(alg); + + // notate based on alg type + bool dopos = false; + switch (alg) { + case CRUSH_BUCKET_UNIFORM: + out << "\t# do not change bucket size (" << n << ") unnecessarily"; + dopos = true; + break; + case CRUSH_BUCKET_LIST: + out << "\t# add new items at the end; do not change order unnecessarily"; + break; + case CRUSH_BUCKET_TREE: + out << "\t# do not change pos for existing items unnecessarily"; + dopos = true; + break; + } + out << "\n"; + + int hash = crush.get_bucket_hash(i); + out << "\thash " << hash << "\t# " << crush_hash_name(hash) << "\n"; + + for (int j=0; j& dcb_states, + ostream &out) +{ + if ((cur == 0) || (!crush.bucket_exists(cur))) + return 0; + + std::map::iterator c = dcb_states.find(cur); + if (c == dcb_states.end()) { + // Mark this bucket as "in progress." + std::map::value_type val(cur, DCB_STATE_IN_PROGRESS); + std::pair ::iterator, bool> rval + (dcb_states.insert(val)); + assert(rval.second); + c = rval.first; + } + else if (c->second == DCB_STATE_DONE) { + // We already did this bucket. + return 0; + } + else if (c->second == DCB_STATE_IN_PROGRESS) { + err << "decompile_crush_bucket: logic error: tried to decompile " + "a bucket that is already being decompiled" << std::endl; + return -EBADE; + } + else { + err << "decompile_crush_bucket: logic error: illegal bucket state! " + << c->second << std::endl; + return -EBADE; + } + + int bsize = crush.get_bucket_size(cur); + for (int i = 0; i < bsize; ++i) { + int item = crush.get_bucket_item(cur, i); + std::map::iterator d = dcb_states.find(item); + if (d == dcb_states.end()) { + int ret = decompile_bucket(item, dcb_states, out); + if (ret) + return ret; + } + else if (d->second == DCB_STATE_IN_PROGRESS) { + err << "decompile_crush_bucket: error: while trying to output bucket " + << cur << ", we found out that it contains one of the buckets that " + << "contain it. This is not allowed. The buckets must form a " + << "directed acyclic graph." << std::endl; + return -EINVAL; + } + else if (d->second != DCB_STATE_DONE) { + err << "decompile_crush_bucket: logic error: illegal bucket state " + << d->second << std::endl; + return -EBADE; + } + } + decompile_bucket_impl(cur, out); + c->second = DCB_STATE_DONE; + return 0; +} + +int CrushCompiler::decompile(ostream &out) +{ + out << "# begin crush map\n\n"; + + out << "# devices\n"; + for (int i=0; i dcb_states; + for (int bucket = -1; bucket > -1-crush.get_max_buckets(); --bucket) { + int ret = decompile_bucket(bucket, dcb_states, out); + if (ret) + return ret; + } + + out << "\n# rules\n"; + for (int i=0; i 0 && + s[0] == ' ') + s = string(s.begin() + 1, s.end()); + return s; +} + +int CrushCompiler::int_node(node_t &node) +{ + string str = string_node(node); + return strtol(str.c_str(), 0, 10); +} + +float CrushCompiler::float_node(node_t &node) +{ + string s = string_node(node); + return strtof(s.c_str(), 0); +} + +int CrushCompiler::parse_device(iter_t const& i) +{ + int id = int_node(i->children[1]); + + string name = string_node(i->children[2]); + crush.set_item_name(id, name.c_str()); + if (item_id.count(name)) { + err << "item " << name << " defined twice" << std::endl; + return -1; + } + item_id[name] = id; + id_item[id] = name; + + if (verbose) err << "device " << id << " '" << name << "'" << std::endl; + return 0; +} + +int CrushCompiler::parse_bucket_type(iter_t const& i) +{ + int id = int_node(i->children[1]); + string name = string_node(i->children[2]); + if (verbose) err << "type " << id << " '" << name << "'" << std::endl; + type_id[name] = id; + crush.set_type_name(id, name.c_str()); + return 0; +} + +int CrushCompiler::parse_bucket(iter_t const& i) +{ + string tname = string_node(i->children[0]); + if (!type_id.count(tname)) { + err << "bucket type '" << tname << "' is not defined" << std::endl; + return -1; + } + int type = type_id[tname]; + + string name = string_node(i->children[1]); + if (item_id.count(name)) { + err << "bucket or device '" << name << "' is already defined" << std::endl; + return -1; + } + + int id = 0; // none, yet! + int alg = -1; + int hash = 0; + set used_items; + int size = 0; + + for (unsigned p=3; pchildren.size()-1; p++) { + iter_t sub = i->children.begin() + p; + string tag = string_node(sub->children[0]); + //err << "tag " << tag << std::endl; + if (tag == "id") + id = int_node(sub->children[1]); + else if (tag == "alg") { + string a = string_node(sub->children[1]); + if (a == "uniform") + alg = CRUSH_BUCKET_UNIFORM; + else if (a == "list") + alg = CRUSH_BUCKET_LIST; + else if (a == "tree") + alg = CRUSH_BUCKET_TREE; + else if (a == "straw") + alg = CRUSH_BUCKET_STRAW; + else { + err << "unknown bucket alg '" << a << "'" << std::endl << std::endl; + return -EINVAL; + } + } + else if (tag == "hash") { + string a = string_node(sub->children[1]); + if (a == "rjenkins1") + hash = CRUSH_HASH_RJENKINS1; + else + hash = atoi(a.c_str()); + } + else if (tag == "item") { + // first, just determine which item pos's are already used + size++; + for (unsigned q = 2; q < sub->children.size(); q++) { + string tag = string_node(sub->children[q++]); + if (tag == "pos") { + int pos = int_node(sub->children[q]); + if (used_items.count(pos)) { + err << "item '" << string_node(sub->children[1]) << "' in bucket '" << name << "' has explicit pos " << pos << ", which is occupied" << std::endl; + return -1; + } + used_items.insert(pos); + } + } + } + else assert(0); + } + + // now do the items. + if (!used_items.empty()) + size = MAX(size, *used_items.rbegin()); + vector items(size); + vector weights(size); + + int curpos = 0; + float bucketweight = 0; + bool have_uniform_weight = false; + float uniform_weight = 0; + for (unsigned p=3; pchildren.size()-1; p++) { + iter_t sub = i->children.begin() + p; + string tag = string_node(sub->children[0]); + if (tag == "item") { + + string iname = string_node(sub->children[1]); + if (!item_id.count(iname)) { + err << "item '" << iname << "' in bucket '" << name << "' is not defined" << std::endl; + return -1; + } + int itemid = item_id[iname]; + + float weight = 1.0; + if (item_weight.count(itemid)) + weight = item_weight[itemid]; + + int pos = -1; + for (unsigned q = 2; q < sub->children.size(); q++) { + string tag = string_node(sub->children[q++]); + if (tag == "weight") + weight = float_node(sub->children[q]); + else if (tag == "pos") + pos = int_node(sub->children[q]); + else + assert(0); + } + if (alg == CRUSH_BUCKET_UNIFORM) { + if (!have_uniform_weight) { + have_uniform_weight = true; + uniform_weight = weight; + } else { + if (uniform_weight != weight) { + err << "item '" << iname << "' in uniform bucket '" << name << "' has weight " << weight + << " but previous item(s) have weight " << uniform_weight + << "; uniform bucket items must all have identical weights." << std::endl; + return -1; + } + } + } + + if (pos >= size) { + err << "item '" << iname << "' in bucket '" << name << "' has pos " << pos << " >= size " << size << std::endl; + return -1; + } + if (pos < 0) { + while (used_items.count(curpos)) curpos++; + pos = curpos++; + } + //err << " item " << iname << " (" << itemid << ") pos " << pos << " weight " << weight << std::endl; + items[pos] = itemid; + weights[pos] = (unsigned)(weight * 0x10000); + bucketweight += weight; + } + } + + if (id == 0) { + for (id=-1; id_item.count(id); id--) ; + //err << "assigned id " << id << std::endl; + } + + if (verbose) err << "bucket " << name << " (" << id << ") " << size << " items and weight " << bucketweight << std::endl; + id_item[id] = name; + item_id[name] = id; + item_weight[id] = bucketweight; + + crush.add_bucket(id, alg, hash, type, size, &items[0], &weights[0]); + crush.set_item_name(id, name.c_str()); + return 0; +} + +int CrushCompiler::parse_rule(iter_t const& i) +{ + int start; // rule name is optional! + + string rname = string_node(i->children[1]); + if (rname != "{") { + if (rule_id.count(rname)) { + err << "rule name '" << rname << "' already defined\n" << std::endl; + return -1; + } + start = 4; + } else { + rname = string(); + start = 3; + } + + int ruleset = int_node(i->children[start]); + + string tname = string_node(i->children[start+2]); + int type; + if (tname == "replicated") + type = CEPH_PG_TYPE_REP; + else if (tname == "raid4") + type = CEPH_PG_TYPE_RAID4; + else + assert(0); + + int minsize = int_node(i->children[start+4]); + int maxsize = int_node(i->children[start+6]); + + int steps = i->children.size() - start - 8; + //err << "num steps " << steps << std::endl; + + int ruleno = crush.add_rule(steps, ruleset, type, minsize, maxsize, -1); + if (rname.length()) { + crush.set_rule_name(ruleno, rname.c_str()); + rule_id[rname] = ruleno; + } + + int step = 0; + for (iter_t p = i->children.begin() + start + 7; step < steps; p++) { + iter_t s = p->children.begin() + 1; + int stepid = s->value.id().to_long(); + switch (stepid) { + case crush_grammar::_step_take: + { + string item = string_node(s->children[1]); + if (!item_id.count(item)) { + err << "in rule '" << rname << "' item '" << item << "' not defined" << std::endl; + return -1; + } + crush.set_rule_step_take(ruleno, step++, item_id[item]); + } + break; + + case crush_grammar::_step_choose: + case crush_grammar::_step_chooseleaf: + { + string type = string_node(s->children[4]); + if (!type_id.count(type)) { + err << "in rule '" << rname << "' type '" << type << "' not defined" << std::endl; + return -1; + } + string choose = string_node(s->children[0]); + string mode = string_node(s->children[1]); + if (choose == "choose") { + if (mode == "firstn") + crush.set_rule_step_choose_firstn(ruleno, step++, int_node(s->children[2]), type_id[type]); + else if (mode == "indep") + crush.set_rule_step_choose_indep(ruleno, step++, int_node(s->children[2]), type_id[type]); + else assert(0); + } else if (choose == "chooseleaf") { + if (mode == "firstn") + crush.set_rule_step_choose_leaf_firstn(ruleno, step++, int_node(s->children[2]), type_id[type]); + else if (mode == "indep") + crush.set_rule_step_choose_leaf_indep(ruleno, step++, int_node(s->children[2]), type_id[type]); + else assert(0); + } else assert(0); + } + break; + + case crush_grammar::_step_emit: + crush.set_rule_step_emit(ruleno, step++); + break; + + default: + err << "bad crush step " << stepid << std::endl; + return -1; + } + } + assert(step == steps); + return 0; +} + +void CrushCompiler::find_used_bucket_ids(iter_t const& i) +{ + for (iter_t p = i->children.begin(); p != i->children.end(); p++) { + if ((int)p->value.id().to_long() == crush_grammar::_bucket) { + iter_t firstline = p->children.begin() + 3; + string tag = string_node(firstline->children[0]); + if (tag == "id") { + int id = int_node(firstline->children[1]); + //err << "saw bucket id " << id << std::endl; + id_item[id] = string(); + } + } + } +} + +int CrushCompiler::parse_crush(iter_t const& i) +{ + find_used_bucket_ids(i); + + int r; + for (iter_t p = i->children.begin(); p != i->children.end(); p++) { + switch (p->value.id().to_long()) { + case crush_grammar::_device: + r = parse_device(p); + break; + case crush_grammar::_bucket_type: + r = parse_bucket_type(p); + break; + case crush_grammar::_bucket: + r = parse_bucket(p); + break; + case crush_grammar::_crushrule: + r = parse_rule(p); + break; + default: + assert(0); + } + } + + if (r < 0) + return r; + + //err << "max_devices " << crush.get_max_devices() << std::endl; + crush.finalize(); + + return 0; +} + +string CrushCompiler::consolidate_whitespace(string in) +{ + string out; + + bool white = false; + for (unsigned p=0; p 3) + err << " \"" << in << "\" -> \"" << out << "\"" << std::endl; + return out; +} + +void CrushCompiler::dump(iter_t const& i, int ind) +{ + err << "dump"; + for (int j=0; jvalue.id().to_long(); + err << id << "\t"; + err << "'" << string(i->value.begin(), i->value.end()) + << "' " << i->children.size() << " children" << std::endl; + for (unsigned int j = 0; j < i->children.size(); j++) + dump(i->children.begin() + j, ind+1); +} + + +int CrushCompiler::compile(istream& in, const char *infn) +{ + if (!infn) + infn = ""; + + string big; + string str; + int line = 1; + map line_pos; // pos -> line + map line_val; + while (getline(in, str)) { + // remove newline + int l = str.length(); + if (l && str[l] == '\n') + str.erase(l-1, 1); + + line_val[line] = str; + + // strip comment + int n = str.find("#"); + if (n >= 0) + str.erase(n, str.length()-n); + + if (verbose>1) err << line << ": " << str << std::endl; + + // work around spirit crankiness by removing extraneous + // whitespace. there is probably a more elegant solution, but + // this only broke with the latest spirit (with the switchover to + // "classic"), i don't want to spend too much time figuring it + // out. + string stripped = consolidate_whitespace(str); + if (stripped.length() && big.length() && big[big.length()-1] != ' ') big += " "; + + line_pos[big.length()] = line; + line++; + big += stripped; + } + + if (verbose > 2) err << "whole file is: \"" << big << "\"" << std::endl; + + crush_grammar crushg; + const char *start = big.c_str(); + //tree_parse_info info = ast_parse(start, crushg, space_p); + tree_parse_info<> info = ast_parse(start, crushg, space_p); + + // parse error? + if (!info.full) { + int cpos = info.stop - start; + //out << "cpos " << cpos << std::endl; + //out << " linemap " << line_pos << std::endl; + assert(!line_pos.empty()); + map::iterator p = line_pos.upper_bound(cpos); + if (p != line_pos.begin()) + p--; + int line = p->second; + int pos = cpos - p->first; + err << infn << ":" << line //<< ":" << (pos+1) + << " error: parse error at '" << line_val[line].substr(pos) << "'" << std::endl; + return -1; + } + + //out << "parsing succeeded\n"; + //dump(info.trees.begin()); + return parse_crush(info.trees.begin()); +} diff --git a/src/crush/CrushCompiler.h b/src/crush/CrushCompiler.h new file mode 100644 index 0000000000000..5476f6b2e810a --- /dev/null +++ b/src/crush/CrushCompiler.h @@ -0,0 +1,65 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_CRUSH_COMPILER_H +#define CEPH_CRUSH_COMPILER_H + +#include "crush/CrushWrapper.h" +#include "crush/grammar.h" + +#include +#include +#include + +class CrushCompiler { + CrushWrapper& crush; + ostream& err; + int verbose; + + // decompile + enum dcb_state_t { + DCB_STATE_IN_PROGRESS = 0, + DCB_STATE_DONE + }; + + int decompile_bucket_impl(int i, ostream &out); + int decompile_bucket(int cur, + std::map& dcb_states, + ostream &out); + + // compile + typedef char const* iterator_t; + typedef tree_match parse_tree_match_t; + typedef parse_tree_match_t::tree_iterator iter_t; + typedef parse_tree_match_t::node_t node_t; + + map item_id; + map id_item; + map item_weight; + map type_id; + map rule_id; + + string string_node(node_t &node); + int int_node(node_t &node); + float float_node(node_t &node); + + int parse_device(iter_t const& i); + int parse_bucket_type(iter_t const& i); + int parse_bucket(iter_t const& i); + int parse_rule(iter_t const& i); + void find_used_bucket_ids(iter_t const& i); + int parse_crush(iter_t const& i); + void dump(iter_t const& i, int ind=1); + + string consolidate_whitespace(string in); + +public: + CrushCompiler(CrushWrapper& c, ostream& eo, int verbosity=0) + : crush(c), err(eo), verbose(verbosity) {} + ~CrushCompiler() {} + + int decompile(ostream& out); + int compile(istream& in, const char *infn=0); +}; + +#endif diff --git a/src/crushtool.cc b/src/crushtool.cc index 9918a6c62fdca..d3d972e151262 100644 --- a/src/crushtool.cc +++ b/src/crushtool.cc @@ -27,747 +27,21 @@ #include "global/global_context.h" #include "global/global_init.h" #include "crush/CrushWrapper.h" -#include "crush/grammar.h" +#include "crush/CrushCompiler.h" - -#include #include -#include -#include -#include -#include -#include - -#include -#ifndef EBADE -#define EBADE EFTYPE -#endif using namespace std; -typedef char const* iterator_t; -typedef tree_match parse_tree_match_t; -typedef parse_tree_match_t::tree_iterator iter_t; -typedef parse_tree_match_t::node_t node_t; - -int verbose = 0; - -map item_id; -map id_item; -map item_weight; - -map type_id; - -map rule_id; - void usage(); -string string_node(node_t &node) -{ - string s = string(node.value.begin(), node.value.end()); - while (s.length() > 0 && - s[0] == ' ') - s = string(s.begin() + 1, s.end()); - return s; -} - -int int_node(node_t &node) -{ - string str = string_node(node); - return strtol(str.c_str(), 0, 10); -} - -float float_node(node_t &node) -{ - string s = string_node(node); - return strtof(s.c_str(), 0); -} - -void parse_device(iter_t const& i, CrushWrapper &crush) -{ - int id = int_node(i->children[1]); - - string name = string_node(i->children[2]); - crush.set_item_name(id, name.c_str()); - if (item_id.count(name)) { - cerr << "item " << name << " defined twice" << std::endl; - exit(1); - } - item_id[name] = id; - id_item[id] = name; - - if (verbose) cout << "device " << id << " '" << name << "'" << std::endl; -} - -void parse_bucket_type(iter_t const& i, CrushWrapper &crush) -{ - int id = int_node(i->children[1]); - string name = string_node(i->children[2]); - if (verbose) cout << "type " << id << " '" << name << "'" << std::endl; - type_id[name] = id; - crush.set_type_name(id, name.c_str()); -} - -void parse_bucket(iter_t const& i, CrushWrapper &crush) -{ - string tname = string_node(i->children[0]); - if (!type_id.count(tname)) { - cerr << "bucket type '" << tname << "' is not defined" << std::endl; - exit(1); - } - int type = type_id[tname]; - - string name = string_node(i->children[1]); - if (item_id.count(name)) { - cerr << "bucket or device '" << name << "' is already defined" << std::endl; - exit(1); - } - - int id = 0; // none, yet! - int alg = -1; - int hash = 0; - set used_items; - int size = 0; - - for (unsigned p=3; pchildren.size()-1; p++) { - iter_t sub = i->children.begin() + p; - string tag = string_node(sub->children[0]); - //cout << "tag " << tag << std::endl; - if (tag == "id") - id = int_node(sub->children[1]); - else if (tag == "alg") { - string a = string_node(sub->children[1]); - if (a == "uniform") - alg = CRUSH_BUCKET_UNIFORM; - else if (a == "list") - alg = CRUSH_BUCKET_LIST; - else if (a == "tree") - alg = CRUSH_BUCKET_TREE; - else if (a == "straw") - alg = CRUSH_BUCKET_STRAW; - else { - cerr << "unknown bucket alg '" << a << "'" << std::endl << std::endl; - usage(); - } - } - else if (tag == "hash") { - string a = string_node(sub->children[1]); - if (a == "rjenkins1") - hash = CRUSH_HASH_RJENKINS1; - else - hash = atoi(a.c_str()); - } - else if (tag == "item") { - // first, just determine which item pos's are already used - size++; - for (unsigned q = 2; q < sub->children.size(); q++) { - string tag = string_node(sub->children[q++]); - if (tag == "pos") { - int pos = int_node(sub->children[q]); - if (used_items.count(pos)) { - cerr << "item '" << string_node(sub->children[1]) << "' in bucket '" << name << "' has explicit pos " << pos << ", which is occupied" << std::endl; - exit(1); - } - used_items.insert(pos); - } - } - } - else assert(0); - } - - // now do the items. - if (!used_items.empty()) - size = MAX(size, *used_items.rbegin()); - vector items(size); - vector weights(size); - - int curpos = 0; - float bucketweight = 0; - bool have_uniform_weight = false; - float uniform_weight = 0; - for (unsigned p=3; pchildren.size()-1; p++) { - iter_t sub = i->children.begin() + p; - string tag = string_node(sub->children[0]); - if (tag == "item") { - - string iname = string_node(sub->children[1]); - if (!item_id.count(iname)) { - cerr << "item '" << iname << "' in bucket '" << name << "' is not defined" << std::endl; - exit(1); - } - int itemid = item_id[iname]; - - float weight = 1.0; - if (item_weight.count(itemid)) - weight = item_weight[itemid]; - - int pos = -1; - for (unsigned q = 2; q < sub->children.size(); q++) { - string tag = string_node(sub->children[q++]); - if (tag == "weight") - weight = float_node(sub->children[q]); - else if (tag == "pos") - pos = int_node(sub->children[q]); - else - assert(0); - } - if (alg == CRUSH_BUCKET_UNIFORM) { - if (!have_uniform_weight) { - have_uniform_weight = true; - uniform_weight = weight; - } else { - if (uniform_weight != weight) { - cerr << "item '" << iname << "' in uniform bucket '" << name << "' has weight " << weight - << " but previous item(s) have weight " << uniform_weight - << "; uniform bucket items must all have identical weights." << std::endl; - exit(1); - } - } - } - - - if (pos >= size) { - cerr << "item '" << iname << "' in bucket '" << name << "' has pos " << pos << " >= size " << size << std::endl; - exit(1); - } - if (pos < 0) { - while (used_items.count(curpos)) curpos++; - pos = curpos++; - } - //cout << " item " << iname << " (" << itemid << ") pos " << pos << " weight " << weight << std::endl; - items[pos] = itemid; - weights[pos] = (unsigned)(weight * 0x10000); - bucketweight += weight; - } - } - - if (id == 0) { - for (id=-1; id_item.count(id); id--) ; - //cout << "assigned id " << id << std::endl; - } - - if (verbose) cout << "bucket " << name << " (" << id << ") " << size << " items and weight " << bucketweight << std::endl; - id_item[id] = name; - item_id[name] = id; - item_weight[id] = bucketweight; - - crush.add_bucket(id, alg, hash, type, size, &items[0], &weights[0]); - crush.set_item_name(id, name.c_str()); -} - -void parse_rule(iter_t const& i, CrushWrapper &crush) -{ - int start; // rule name is optional! - - string rname = string_node(i->children[1]); - if (rname != "{") { - if (rule_id.count(rname)) { - cerr << "rule name '" << rname << "' already defined\n" << std::endl; - exit(1); - } - start = 4; - } else { - rname = string(); - start = 3; - } - - int ruleset = int_node(i->children[start]); - - string tname = string_node(i->children[start+2]); - int type; - if (tname == "replicated") - type = CEPH_PG_TYPE_REP; - else if (tname == "raid4") - type = CEPH_PG_TYPE_RAID4; - else - assert(0); - - int minsize = int_node(i->children[start+4]); - int maxsize = int_node(i->children[start+6]); - - int steps = i->children.size() - start - 8; - //cout << "num steps " << steps << std::endl; - - int ruleno = crush.add_rule(steps, ruleset, type, minsize, maxsize, -1); - if (rname.length()) { - crush.set_rule_name(ruleno, rname.c_str()); - rule_id[rname] = ruleno; - } - - int step = 0; - for (iter_t p = i->children.begin() + start + 7; step < steps; p++) { - iter_t s = p->children.begin() + 1; - int stepid = s->value.id().to_long(); - switch (stepid) { - case crush_grammar::_step_take: - { - string item = string_node(s->children[1]); - if (!item_id.count(item)) { - cerr << "in rule '" << rname << "' item '" << item << "' not defined" << std::endl; - exit(1); - } - crush.set_rule_step_take(ruleno, step++, item_id[item]); - } - break; - - case crush_grammar::_step_choose: - case crush_grammar::_step_chooseleaf: - { - string type = string_node(s->children[4]); - if (!type_id.count(type)) { - cerr << "in rule '" << rname << "' type '" << type << "' not defined" << std::endl; - exit(1); - } - string choose = string_node(s->children[0]); - string mode = string_node(s->children[1]); - if (choose == "choose") { - if (mode == "firstn") - crush.set_rule_step_choose_firstn(ruleno, step++, int_node(s->children[2]), type_id[type]); - else if (mode == "indep") - crush.set_rule_step_choose_indep(ruleno, step++, int_node(s->children[2]), type_id[type]); - else assert(0); - } else if (choose == "chooseleaf") { - if (mode == "firstn") - crush.set_rule_step_choose_leaf_firstn(ruleno, step++, int_node(s->children[2]), type_id[type]); - else if (mode == "indep") - crush.set_rule_step_choose_leaf_indep(ruleno, step++, int_node(s->children[2]), type_id[type]); - else assert(0); - } else assert(0); - } - break; - - case crush_grammar::_step_emit: - crush.set_rule_step_emit(ruleno, step++); - break; - - default: - cerr << "bad crush step " << stepid << std::endl; - assert(0); - } - } - assert(step == steps); -} - -void dump(iter_t const& i, int ind=1) -{ - cout << "dump"; - for (int j=0; jvalue.id().to_long(); - cout << id << "\t"; - cout << "'" << string(i->value.begin(), i->value.end()) - << "' " << i->children.size() << " children" << std::endl; - for (unsigned int j = 0; j < i->children.size(); j++) - dump(i->children.begin() + j, ind+1); -} - -void find_used_bucket_ids(iter_t const& i) -{ - for (iter_t p = i->children.begin(); p != i->children.end(); p++) { - if ((int)p->value.id().to_long() == crush_grammar::_bucket) { - iter_t firstline = p->children.begin() + 3; - string tag = string_node(firstline->children[0]); - if (tag == "id") { - int id = int_node(firstline->children[1]); - //cout << "saw bucket id " << id << std::endl; - id_item[id] = string(); - } - } - } -} - -void parse_crush(iter_t const& i, CrushWrapper &crush) -{ - find_used_bucket_ids(i); - - for (iter_t p = i->children.begin(); p != i->children.end(); p++) { - switch (p->value.id().to_long()) { - case crush_grammar::_device: - parse_device(p, crush); - break; - case crush_grammar::_bucket_type: - parse_bucket_type(p, crush); - break; - case crush_grammar::_bucket: - parse_bucket(p, crush); - break; - case crush_grammar::_crushrule: - parse_rule(p, crush); - break; - default: - assert(0); - } - } - - //cout << "max_devices " << crush.get_max_devices() << std::endl; - crush.finalize(); - -} const char *infn = "stdin"; //////////////////////////////////////////////////////////////////////////// -string consolidate_whitespace(string in) -{ - string out; - - bool white = false; - for (unsigned p=0; p 3) - cout << " \"" << in << "\" -> \"" << out << "\"" << std::endl; - return out; -} - -int compile_crush_file(const char *infn, CrushWrapper &crush) -{ - // read the file - ifstream in(infn); - if (!in.is_open()) { - cerr << "input file " << infn << " not found" << std::endl; - return -ENOENT; - } - - string big; - string str; - int line = 1; - map line_pos; // pos -> line - map line_val; - while (getline(in, str)) { - // remove newline - int l = str.length(); - if (l && str[l] == '\n') - str.erase(l-1, 1); - - line_val[line] = str; - - // strip comment - int n = str.find("#"); - if (n >= 0) - str.erase(n, str.length()-n); - - if (verbose>1) cout << line << ": " << str << std::endl; - - // work around spirit crankiness by removing extraneous - // whitespace. there is probably a more elegant solution, but - // this only broke with the latest spirit (with the switchover to - // "classic"), i don't want to spend too much time figuring it - // out. - string stripped = consolidate_whitespace(str); - if (stripped.length() && big.length() && big[big.length()-1] != ' ') big += " "; - - line_pos[big.length()] = line; - line++; - big += stripped; - } - - if (verbose > 2) cout << "whole file is: \"" << big << "\"" << std::endl; - - crush_grammar crushg; - const char *start = big.c_str(); - //tree_parse_info info = ast_parse(start, crushg, space_p); - tree_parse_info<> info = ast_parse(start, crushg, space_p); - - // parse error? - if (!info.full) { - int cpos = info.stop - start; - //cout << "cpos " << cpos << std::endl; - //cout << " linemap " << line_pos << std::endl; - assert(!line_pos.empty()); - map::iterator p = line_pos.upper_bound(cpos); - if (p != line_pos.begin()) p--; - int line = p->second; - int pos = cpos - p->first; - cerr << infn << ":" << line //<< ":" << (pos+1) - << " error: parse error at '" << line_val[line].substr(pos) << "'" << std::endl; - return -1; - } - - //cout << "parsing succeeded\n"; - //dump(info.trees.begin()); - parse_crush(info.trees.begin(), crush); - - return 0; -} - -void print_type_name(ostream& out, int t, CrushWrapper &crush) -{ - const char *name = crush.get_type_name(t); - if (name) - out << name; - else if (t == 0) - out << "device"; - else - out << "type" << t; -} - -void print_item_name(ostream& out, int t, CrushWrapper &crush) -{ - const char *name = crush.get_item_name(t); - if (name) - out << name; - else if (t >= 0) - out << "device" << t; - else - out << "bucket" << (-1-t); -} - -void print_rule_name(ostream& out, int t, CrushWrapper &crush) -{ - const char *name = crush.get_rule_name(t); - if (name) - out << name; - else - out << "rule" << t; -} - -void print_fixedpoint(ostream& out, int i) -{ - char s[20]; - snprintf(s, sizeof(s), "%.3f", (float)i / (float)0x10000); - out << s; -} - -enum dcb_state_t { - DCB_STATE_IN_PROGRESS = 0, - DCB_STATE_DONE -}; - -static int decompile_crush_bucket_impl(int i, - CrushWrapper &crush, ostream &out) -{ - int type = crush.get_bucket_type(i); - print_type_name(out, type, crush); - out << " "; - print_item_name(out, i, crush); - out << " {\n"; - out << "\tid " << i << "\t\t# do not change unnecessarily\n"; - - out << "\t# weight "; - print_fixedpoint(out, crush.get_bucket_weight(i)); - out << "\n"; - - int n = crush.get_bucket_size(i); - - int alg = crush.get_bucket_alg(i); - out << "\talg " << crush_bucket_alg_name(alg); - - // notate based on alg type - bool dopos = false; - switch (alg) { - case CRUSH_BUCKET_UNIFORM: - out << "\t# do not change bucket size (" << n << ") unnecessarily"; - dopos = true; - break; - case CRUSH_BUCKET_LIST: - out << "\t# add new items at the end; do not change order unnecessarily"; - break; - case CRUSH_BUCKET_TREE: - out << "\t# do not change pos for existing items unnecessarily"; - dopos = true; - break; - } - out << "\n"; - - int hash = crush.get_bucket_hash(i); - out << "\thash " << hash << "\t# " << crush_hash_name(hash) << "\n"; - - for (int j=0; j &dcb_states, - CrushWrapper &crush, ostream &out) -{ - if ((cur == 0) || (!crush.bucket_exists(cur))) - return 0; - - std::map::iterator c = dcb_states.find(cur); - if (c == dcb_states.end()) { - // Mark this bucket as "in progress." - std::map::value_type val(cur, DCB_STATE_IN_PROGRESS); - std::pair ::iterator, bool> rval - (dcb_states.insert(val)); - assert(rval.second); - c = rval.first; - } - else if (c->second == DCB_STATE_DONE) { - // We already did this bucket. - return 0; - } - else if (c->second == DCB_STATE_IN_PROGRESS) { - cout << "decompile_crush_bucket: logic error: tried to decompile " - "a bucket that is already being decompiled" << std::endl; - return -EBADE; - } - else { - cout << "decompile_crush_bucket: logic error: illegal bucket state! " - << c->second << std::endl; - return -EBADE; - } - - int bsize = crush.get_bucket_size(cur); - for (int i = 0; i < bsize; ++i) { - int item = crush.get_bucket_item(cur, i); - std::map::iterator d = dcb_states.find(item); - if (d == dcb_states.end()) { - int ret = decompile_crush_bucket(item, dcb_states, crush, out); - if (ret) - return ret; - } - else if (d->second == DCB_STATE_IN_PROGRESS) { - cout << "decompile_crush_bucket: error: while trying to output bucket " - << cur << ", we found out that it contains one of the buckets that " - << "contain it. This is not allowed. The buckets must form a " - << "directed acyclic graph." << std::endl; - return -EINVAL; - } - else if (d->second != DCB_STATE_DONE) { - cout << "decompile_crush_bucket: logic error: illegal bucket state " - << d->second << std::endl; - return -EBADE; - } - } - decompile_crush_bucket_impl(cur, crush, out); - c->second = DCB_STATE_DONE; - return 0; -} - -int decompile_crush(CrushWrapper &crush, ostream &out) -{ - out << "# begin crush map\n\n"; - - out << "# devices\n"; - for (int i=0; i dcb_states; - for (int bucket = -1; bucket > -1-crush.get_max_buckets(); --bucket) { - int ret = decompile_crush_bucket(bucket, dcb_states, crush, out); - if (ret) - return ret; - } - - out << "\n# rules\n"; - for (int i=0; i